User Inputs

output.var = params$output.var 

transform.abs = FALSE
log.pred = params$log.pred
norm.pred = FALSE
algo.forward.caret = params$algo.forward.caret
algo.backward.caret = params$algo.backward.caret
algo.stepwise.caret = params$algo.stepwise.caret
algo.LASSO.caret = params$algo.LASSO.caret
algo.LARS.caret = params$algo.LARS.caret
message("Parameters used for training/prediction: ")
## Parameters used for training/prediction:
str(params)
## List of 7
##  $ output.var         : chr "y3"
##  $ log.pred           : logi TRUE
##  $ algo.forward.caret : logi TRUE
##  $ algo.backward.caret: logi TRUE
##  $ algo.stepwise.caret: logi TRUE
##  $ algo.LASSO.caret   : logi TRUE
##  $ algo.LARS.caret    : logi TRUE
# Setup Labels
output.var.tr = if (log.pred == TRUE)  paste0(output.var,'.log') else  output.var.tr = output.var

Loading Data

feat  = read.csv('../../Data/features_highprec.csv')
labels = read.csv('../../Data/labels.csv')
predictors = names(dplyr::select(feat,-JobName))
data.ori = inner_join(feat,labels,by='JobName')
#data.ori = inner_join(feat,select_at(labels,c('JobName',output.var)),by='JobName')

Data validation

cc  = complete.cases(data.ori)
data.notComplete = data.ori[! cc,]
data = data.ori[cc,] %>% select_at(c(predictors,output.var,'JobName'))
message('Original cases: ',nrow(data.ori))
## Original cases: 10000
message('Non-Complete cases: ',nrow(data.notComplete))
## Non-Complete cases: 3020
message('Complete cases: ',nrow(data))
## Complete cases: 6980
summary(dplyr::select_at(data,c('JobName',output.var)))
##       JobName           y3        
##  Job_00001:   1   Min.   : 95.91  
##  Job_00002:   1   1st Qu.:118.29  
##  Job_00003:   1   Median :124.03  
##  Job_00004:   1   Mean   :125.40  
##  Job_00007:   1   3rd Qu.:131.06  
##  Job_00008:   1   Max.   :193.73  
##  (Other)  :6974

Output Variable

The Output Variable y3 shows right skewness, so will proceed with a log transformation

Histogram

df=gather(select_at(data,output.var))
ggplot(df, aes(x=value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() 

  #stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  

QQPlot

ggplot(gather(select_at(data,output.var)), aes(sample=value)) + 
  stat_qq() + 
  facet_wrap(~key, scales = 'free',ncol=4)

Trasformation of Output Variable from y3 to y3.log

if(log.pred==TRUE) data[[output.var.tr]] = log(data[[output.var]],10) else
  data[[output.var.tr]] = data[[output.var]]
df=gather(select_at(data,c(output.var,output.var.tr)))
ggplot(df, aes(value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() + 
  # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
  facet_wrap(~key, scales = 'free',ncol=2)

ggplot(gather(select_at(data,c(output.var,output.var.tr))), aes(sample=value)) + 
  stat_qq() + 
  facet_wrap(~key, scales = 'free',ncol=4)

Best Normalizator y3

Normalization of y3 using bestNormalize package. (suggested orderNorm) This is cool, but I think is too far for the objective of the project

t=bestNormalize::bestNormalize(data[[output.var]])
t
## Best Normalizing transformation with 6980 Observations
##  Estimated Normality Statistics (Pearson P / df, lower => more normal):
##  - No transform: 2.9322 
##  - Box-Cox: 1.4024 
##  - Log_b(x+a): 1.9956 
##  - sqrt(x+a): 2.432 
##  - exp(x): 749.2818 
##  - arcsinh(x): 1.9956 
##  - Yeo-Johnson: 1.1845 
##  - orderNorm: 1.1231 
## Estimation method: Out-of-sample via CV with 10 folds and 5 repeats
##  
## Based off these, bestNormalize chose:
## orderNorm Transformation with 6980 nonmissing obs and no ties 
##  - Original quantiles:
##      0%     25%     50%     75%    100% 
##  95.913 118.289 124.030 131.059 193.726
qqnorm(data[[output.var]])

qqnorm(predict(t))

orderNorm() is a rank-based procedure by which the values of a vector are mapped to their percentile, which is then mapped to the same percentile of the normal distribution. Without the presence of ties, this essentially guarantees that the transformation leads to a uniform distribution

Predictors

All predictors show a Fat-Tail situation, where the two tails are very tall, and a low distribution around the mean. The orderNorm transformation can help (see [Best Normalizator] section)

Interesting Predictors

Histograms

cols = c('x11','x18','stat98','x7','stat110')
df=gather(select_at(data,cols))
ggplot(df, aes(value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() + 
  # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
  facet_wrap(~key, scales = 'free',ncol=3)

# ggplot(gather(select_at(data,cols)), aes(sample=value)) + 
#   stat_qq()+
#   facet_wrap(~key, scales = 'free',ncol=2)

lapply(select_at(data,cols),summary)
## $x11
##      Min.   1st Qu.    Median      Mean   3rd Qu.      Max. 
## 9.000e-08 9.494e-08 1.001e-07 1.001e-07 1.052e-07 1.100e-07 
## 
## $x18
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   1.500   3.147   4.769   4.772   6.418   7.999 
## 
## $stat98
##      Min.   1st Qu.    Median      Mean   3rd Qu.      Max. 
## -2.998619 -1.551882 -0.015993 -0.005946  1.528405  2.999499 
## 
## $x7
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   0.700   1.266   1.854   1.852   2.446   3.000 
## 
## $stat110
##      Min.   1st Qu.    Median      Mean   3rd Qu.      Max. 
## -2.999543 -1.496865 -0.002193 -0.004129  1.504273  2.999563

Scatter plot vs. output variable **y3.log

d = gather(dplyr::select_at(data,c(cols,output.var.tr)),key=target,value=value,-!!output.var.tr)
ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
  geom_point(color='light green',alpha=0.5) + 
  geom_smooth() + 
  facet_wrap(~target, scales = 'free',ncol=3)
## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'

All Predictors

Histograms

All indicators have a strong indication of Fat-Tails

df=gather(select_at(data,predictors))
ggplot(df, aes(value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() + 
  # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
  facet_wrap(~key, scales = 'free',ncol=4)

Correlations

With Output Variable

#chart.Correlation(select(data,-JobName),  pch=21)
t=as.data.frame(round(cor(dplyr::select(data,-one_of(output.var.tr,'JobName'))
                          ,select_at(data,output.var.tr)),4))  %>%
  rownames_to_column(var='variable') %>% filter(variable != !!output.var) %>% arrange(-y3.log)
#DT::datatable(t)
message("Top Positive")
## Top Positive
kable(head(arrange(t,desc(y3.log)),20))
variable y3.log
x18 0.3120
x7 0.2091
stat98 0.1784
x9 0.1127
x17 0.0611
x16 0.0489
x10 0.0472
x21 0.0412
x11 0.0322
x8 0.0318
stat156 0.0287
stat23 0.0234
stat100 0.0206
stat144 0.0203
stat59 0.0202
stat60 0.0199
stat195 0.0199
stat141 0.0194
stat73 0.0192
stat197 0.0185
message("Top Negative")
## Top Negative
kable(head(arrange(t,y3.log),20))
variable y3.log
stat110 -0.1594
x4 -0.0603
stat13 -0.0345
stat41 -0.0345
stat14 -0.0317
stat149 -0.0309
stat113 -0.0279
stat4 -0.0248
stat106 -0.0236
stat146 -0.0236
stat186 -0.0217
stat91 -0.0210
stat214 -0.0209
stat5 -0.0207
stat22 -0.0202
stat39 -0.0202
stat175 -0.0194
stat187 -0.0193
stat128 -0.0192
stat37 -0.0191

Between All Variables

#chart.Correlation(select(data,-JobName),  pch=21)
t=as.data.frame(round(cor(dplyr::select(data,-one_of('JobName'))),4))
#DT::datatable(t,options=list(scrollX=T))
message("Showing only 10 variables")
## Showing only 10 variables
kable(t[1:10,1:10])
x1 x2 x3 x4 x5 x6 x7 x8 x9 x10
x1 1.0000 0.0034 -0.0028 0.0085 0.0068 0.0159 0.0264 -0.0012 0.0142 0.0013
x2 0.0034 1.0000 -0.0057 0.0004 -0.0094 -0.0101 0.0089 0.0078 0.0049 -0.0214
x3 -0.0028 -0.0057 1.0000 0.0029 0.0046 0.0006 -0.0105 -0.0002 0.0167 -0.0137
x4 0.0085 0.0004 0.0029 1.0000 -0.0059 0.0104 0.0098 0.0053 0.0061 -0.0023
x5 0.0068 -0.0094 0.0046 -0.0059 1.0000 0.0016 -0.0027 0.0081 0.0259 -0.0081
x6 0.0159 -0.0101 0.0006 0.0104 0.0016 1.0000 0.0200 -0.0157 0.0117 -0.0072
x7 0.0264 0.0089 -0.0105 0.0098 -0.0027 0.0200 1.0000 -0.0018 -0.0069 -0.0221
x8 -0.0012 0.0078 -0.0002 0.0053 0.0081 -0.0157 -0.0018 1.0000 0.0142 -0.0004
x9 0.0142 0.0049 0.0167 0.0061 0.0259 0.0117 -0.0069 0.0142 1.0000 0.0149
x10 0.0013 -0.0214 -0.0137 -0.0023 -0.0081 -0.0072 -0.0221 -0.0004 0.0149 1.0000

Scatter Plots with Output Variable

Scatter plots with all predictors and the output variable (y3.log)

d = gather(dplyr::select_at(data,c(predictors,output.var.tr)),key=target,value=value,-!!output.var.tr)
ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
  geom_point(color='light blue',alpha=0.5) + 
  geom_smooth() + 
  facet_wrap(~target, scales = 'free',ncol=4)
## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'

Multicollinearity - VIF

No Multicollinearity among predictors

Showing Top predictor by VIF Value

vifDF = usdm::vif(select_at(data,predictors)) %>% arrange(desc(VIF))
head(vifDF,15)
##    Variables      VIF
## 1    stat142 1.062287
## 2    stat202 1.060639
## 3    stat204 1.060269
## 4     stat31 1.059967
## 5    stat175 1.059217
## 6    stat127 1.059183
## 7    stat138 1.058689
## 8    stat164 1.058262
## 9    stat178 1.058254
## 10   stat154 1.058008
## 11    stat20 1.057994
## 12   stat113 1.057933
## 13   stat114 1.057876
## 14   stat169 1.057705
## 15    stat97 1.057519

Feature Eng

  • Square Root transformation for x18
data.tr=data %>%
  mutate(x18.sqrt = sqrt(x18)) 
cols=c('x18','x18.sqrt')

Comparing Pre and Post Transformation Density Plots

# ggplot(gather(select_at(data.tr,cols)), aes(value)) + 
#   geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
#   geom_density() + 
#   facet_wrap(~key, scales = 'free',ncol=4)

d = gather(dplyr::select_at(data.tr,c(cols,output.var.tr)),key=target,value=value,-!!output.var.tr)
ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
  geom_point(color='light blue',alpha=0.5) + 
  geom_smooth() + 
  facet_wrap(~target, scales = 'free',ncol=4)
## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'

#removing unwanted variables
data.tr=data.tr %>%
  dplyr::select_at(names(data.tr)[! names(data.tr) %in% c('x18','y3','JobName')])

data=data.tr
label.names=output.var.tr

Modeling

Train Test Split

data = data[sample(nrow(data)),] # randomly shuffle data
split = sample.split(data[,label.names], SplitRatio = 0.8)

data.train = subset(data, split == TRUE)
data.test = subset(data, split == FALSE)

Common Functions

plot.diagnostics <-  function(model, train) {
  plot(model)
  
  residuals = resid(model) # Plotted above in plot(lm.out)
  r.standard = rstandard(model)
  r.student = rstudent(model)
  
  df = data.frame(x=predict(model,train),y=r.student)
  p=ggplot(data=df,aes(x=x,y=y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_hline(yintercept = 0,size=1)+
    ylab("Student Residuals") +
    xlab("Predicted Values")+
    ggtitle("Student Residual Plot")
  plot(p)
  
  df = data.frame(x=predict(model,train),y=r.standard)
  p=ggplot(data=df,aes(x=x,y=y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_hline(yintercept = c(-2,0,2),size=1)+
    ylab("Student Residuals") +
    xlab("Predicted Values")+
    ggtitle("Student Residual Plot")
  plot(p)
  # Histogram
  df=data.frame(r.student)
  p=ggplot(data=df,aes(r.student)) +
    geom_histogram(aes(y=..density..),bins = 50,fill='blue',alpha=0.6) + 
    stat_function(fun = dnorm, n = 100, args = list(mean = 0, sd = 1)) +
    ylab("Density")+
    xlab("Studentized Residuals")+
    ggtitle("Distribution of Studentized Residuals")
  plot(p)
  # http://www.stat.columbia.edu/~martin/W2024/R7.pdf
  # Influential plots
  inf.meas = influence.measures(model)
  # print (summary(inf.meas)) # too much data
  
  # Leverage plot
  lev = hat(model.matrix(model))
  df=tibble::rownames_to_column(as.data.frame(lev),'id')
  p=ggplot(data=df,aes(x=as.numeric(id),y=lev)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    ylab('Leverage - check') + 
    xlab('Index')
  plot(p)
  # Cook's Distance
  cd = cooks.distance(model)
  df=tibble::rownames_to_column(as.data.frame(cd),'id')
  p=ggplot(data=df,aes(x=as.numeric(id),y=cd)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_text(data=filter(df,cd>15/nrow(train)),aes(label=id),check_overlap=T,size=3,vjust=-.5)+
    ylab('Cooks distances') + 
    geom_hline(yintercept = c(4/nrow(train),0),size=1)+
    xlab('Index')
  plot(p)
  print (paste("Number of data points that have Cook's D > 4/n: ", length(cd[cd > 4/nrow(train)]), sep = "")) 
  print (paste("Number of data points that have Cook's D > 1: ", length(cd[cd > 1]), sep = "")) 
  return(cd)
}

# function to set up random seeds
# Based on http://jaehyeon-kim.github.io/2015/05/Setup-Random-Seeds-on-Caret-Package.html 
setCaretSeeds <- function(method = "cv", numbers = 1, repeats = 1, tunes = NULL, seed = 1701) {
  #B is the number of resamples and integer vector of M (numbers + tune length if any)
  B <- if (method == "cv") numbers
  else if(method == "repeatedcv") numbers * repeats
  else NULL
  if(is.null(length)) {
    seeds <- NULL
  } else {
    set.seed(seed = seed)
    seeds <- vector(mode = "list", length = B)
    seeds <- lapply(seeds, function(x) sample.int(n = 1000000
                                                  , size = numbers + ifelse(is.null(tunes), 0, tunes)))
    seeds[[length(seeds) + 1]] <- sample.int(n = 1000000, size = 1)
  }
  # return seeds
  seeds
}

train.caret.glmselect = function(formula, data, method
                                 ,subopt = NULL, feature.names
                                 , train.control = NULL, tune.grid = NULL, pre.proc = NULL){
  
  if(is.null(train.control)){
    train.control <- trainControl(method = "cv"
                              ,number = 10
                              ,seeds = setCaretSeeds(method = "cv"
                                                     , numbers = 10
                                                     , seed = 1701)
                              ,search = "grid"
                              ,verboseIter = TRUE
                              ,allowParallel = TRUE
                              )
  }
  
  if(is.null(tune.grid)){
    if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
      tune.grid = data.frame(nvmax = 1:length(feature.names))
    }
    if (method == 'glmnet' && subopt == 'LASSO'){
      # Will only show 1 Lambda value during training, but that is OK
      # https://stackoverflow.com/questions/47526544/why-need-to-tune-lambda-with-carettrain-method-glmnet-and-cv-glmnet
      # Another option for LASSO is this: https://github.com/topepo/caret/blob/master/RegressionTests/Code/lasso.R
      lambda = 10^seq(-2,0, length =100)
      alpha = c(1)
      tune.grid = expand.grid(alpha = alpha,lambda = lambda)
    }
    if (method == 'lars'){
      # https://github.com/topepo/caret/blob/master/RegressionTests/Code/lars.R
      fraction = seq(0, 1, length = 100)
      tune.grid = expand.grid(fraction = fraction)
      pre.proc = c("center", "scale") 
    }
  }
  
  # http://sshaikh.org/2015/05/06/parallelize-machine-learning-in-r-with-multi-core-cpus/
  cl <- makeCluster(ceiling(detectCores()*0.85)) # use 75% of cores only, leave rest for other tasks
  registerDoParallel(cl)

  set.seed(1) 
  # note that the seed has to actually be set just before this function is called
  # settign is above just not ensure reproducibility for some reason
  model.caret <- caret::train(formula
                              , data = data
                              , method = method
                              , tuneGrid = tune.grid
                              , trControl = train.control
                              , preProc = pre.proc
                              )
  
  stopCluster(cl)
  registerDoSEQ() # register sequential engine in case you are not using this function anymore
  
  if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
    print("All models results")
    print(model.caret$results) # all model results
    print("Best Model")
    print(model.caret$bestTune) # best model
    model = model.caret$finalModel

    # Metrics Plot 
    dataPlot = model.caret$results %>%
      gather(key='metric',value='value',-nvmax) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=nvmax,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot
    # leap function does not support studentized residuals
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)
   
    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') + 
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    id = rownames(model.caret$bestTune)    
    # Provides the coefficients of the best model
    # regsubsets doens return a full model (see documentation of regsubset), so we need to recalcualte themodel
    # https://stackoverflow.com/questions/13063762/how-to-obtain-a-lm-object-from-regsubsets
    print("Coefficients of final model:")
    coefs <- coef(model, id=id)
    #calculate the model to the the coef intervals
    nams <- names(coefs)
    nams <- nams[!nams %in% "(Intercept)"]
    response <-  as.character(formula[[2]])
    form <- as.formula(paste(response, paste(nams, collapse = " + "), sep = " ~ "))
    mod <- lm(form, data = data)
    #coefs
    #coef(mod)
    print(car::Confint(mod))
    return(list(model = model,id = id, residPlot = residPlot, residHistogram=residHistogram
                ,modelLM=mod))
  }
  if (method == 'glmnet' && subopt == 'LASSO'){
    print(model.caret)
    print(plot(model.caret))
    print(model.caret$bestTune)
    
    print(model.caret$results)
    model=model.caret$finalModel
    # Metrics Plot 
    dataPlot = model.caret$results %>%
      gather(key='metric',value='value',-lambda) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=lambda,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot 
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)

    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') +
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    
    print("Coefficients") 
    #no interval for glmnet: https://stackoverflow.com/questions/39750965/confidence-intervals-for-ridge-regression
    t=coef(model,s=model.caret$bestTune$lambda)
    model.coef = t[which(t[,1]!=0),]
    print(as.data.frame(model.coef))
    id = NULL # not really needed but added for consistency
    return(list(model = model.caret,id = id, residPlot = residPlot, metricsPlot=metricsPlot ))
  }
  if (method == 'lars'){
    print(model.caret)
    print(plot(model.caret))
    print(model.caret$bestTune)
    
    # Metrics Plot
    dataPlot = model.caret$results %>%
        gather(key='metric',value='value',-fraction) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=fraction,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)

    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') + 
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    
    print("Coefficients") 
    t=coef(model.caret$finalModel,s=model.caret$bestTune$fraction,mode='fraction')
    model.coef = t[which(t!=0)]
    print(model.coef)
    id = NULL # not really needed but added for consistency
    return(list(model = model.caret,id = id, residPlot = residPlot, residHistogram=residHistogram))
  }
}

# https://stackoverflow.com/questions/48265743/linear-model-subset-selection-goodness-of-fit-with-k-fold-cross-validation
# changed slightly since call[[2]] was just returning "formula" without actually returnign the value in formula
predict.regsubsets <- function(object, newdata, id, formula, ...) {
    #form <- as.formula(object$call[[2]])
    mat <- model.matrix(formula, newdata) # adds intercept and expands any interaction terms
    coefi <- coef(object, id = id)
    xvars <- names(coefi)
    return(mat[,xvars]%*%coefi)
}
  
test.model = function(model, test, level=0.95
                      ,draw.limits = FALSE, good = 0.1, ok = 0.15
                      ,method = NULL, subopt = NULL
                      ,id = NULL, formula, feature.names, label.names
                      ,transformation = NULL){
  ## if using caret for glm select equivalent functionality, 
  ## need to pass formula (full is ok as it will select subset of variables from there)
  if (is.null(method)){
    pred = predict(model, newdata=test, interval="confidence", level = level) 
  }
  
  if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
    pred = predict.regsubsets(model, newdata = test, id = id, formula = formula)
  }
  
  if (method == 'glmnet' && subopt == 'LASSO'){
    xtest = as.matrix(test[,feature.names]) 
    pred=as.data.frame(predict(model, xtest))
  }
  
  if (method == 'lars'){
    pred=as.data.frame(predict(model, newdata = test))
  }
    
  # Summary of predicted values
  print ("Summary of predicted values: ")
  print(summary(pred[,1]))

  test.mse = mean((test[,label.names]-pred[,1])^2)
  print (paste(method, subopt, "Test MSE:", test.mse, sep=" "))
  
  if(log.pred == TRUE || norm.pred == TRUE){
    # plot transformewd comparison first
    df=data.frame(x=test[,label.names],y=pred[,1])
    ggplot(df,aes(x=x,y=y)) +
      geom_point(color='blue',alpha=0.5,shape=20,size=2) +
      geom_abline(slope=1,intercept=0,color='black',size=1) +
      #scale_y_continuous(limits=c(min(df),max(df)))+
      xlab("Actual (Transformed)")+
      ylab("Predicted (Transformed)")
  }
    
  if (log.pred == FALSE && norm.pred == FALSE){
    x = test[,label.names]
    y = pred[,1]
  }
  if (log.pred == TRUE){
    x = 10^test[,label.names]
    y = 10^pred[,1]  
  }
  if (norm.pred == TRUE){
    x = predict(transformation, test[,label.names], inverse = TRUE)
    y = predict(transformation, pred[,1], inverse = TRUE)
  }

  df=data.frame(x,y)
  ggplot(df,aes(x,y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_abline(slope=c(1+good,1-good,1+ok,1-ok)
                ,intercept=rep(0,4),color=c('dark green','dark green','dark red','dark red'),size=1,alpha=0.8) +
    #scale_y_continuous(limits=c(min(df),max(df)))+
    xlab("Actual")+
    ylab("Predicted") 
    
 
}

Setup Formulae

n <- names(data.train)
 formula <- as.formula(paste(paste(n[n %in% label.names], collapse = " + ")
                             ," ~", paste(n[!n %in% label.names], collapse = " + "))) 

grand.mean.formula = as.formula(paste(paste(n[n %in% label.names], collapse = " + ")," ~ 1"))

print(formula)
## y3.log ~ x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + 
##     x12 + x13 + x14 + x15 + x16 + x17 + x19 + x20 + x21 + x22 + 
##     x23 + stat1 + stat2 + stat3 + stat4 + stat5 + stat6 + stat7 + 
##     stat8 + stat9 + stat10 + stat11 + stat12 + stat13 + stat14 + 
##     stat15 + stat16 + stat17 + stat18 + stat19 + stat20 + stat21 + 
##     stat22 + stat23 + stat24 + stat25 + stat26 + stat27 + stat28 + 
##     stat29 + stat30 + stat31 + stat32 + stat33 + stat34 + stat35 + 
##     stat36 + stat37 + stat38 + stat39 + stat40 + stat41 + stat42 + 
##     stat43 + stat44 + stat45 + stat46 + stat47 + stat48 + stat49 + 
##     stat50 + stat51 + stat52 + stat53 + stat54 + stat55 + stat56 + 
##     stat57 + stat58 + stat59 + stat60 + stat61 + stat62 + stat63 + 
##     stat64 + stat65 + stat66 + stat67 + stat68 + stat69 + stat70 + 
##     stat71 + stat72 + stat73 + stat74 + stat75 + stat76 + stat77 + 
##     stat78 + stat79 + stat80 + stat81 + stat82 + stat83 + stat84 + 
##     stat85 + stat86 + stat87 + stat88 + stat89 + stat90 + stat91 + 
##     stat92 + stat93 + stat94 + stat95 + stat96 + stat97 + stat98 + 
##     stat99 + stat100 + stat101 + stat102 + stat103 + stat104 + 
##     stat105 + stat106 + stat107 + stat108 + stat109 + stat110 + 
##     stat111 + stat112 + stat113 + stat114 + stat115 + stat116 + 
##     stat117 + stat118 + stat119 + stat120 + stat121 + stat122 + 
##     stat123 + stat124 + stat125 + stat126 + stat127 + stat128 + 
##     stat129 + stat130 + stat131 + stat132 + stat133 + stat134 + 
##     stat135 + stat136 + stat137 + stat138 + stat139 + stat140 + 
##     stat141 + stat142 + stat143 + stat144 + stat145 + stat146 + 
##     stat147 + stat148 + stat149 + stat150 + stat151 + stat152 + 
##     stat153 + stat154 + stat155 + stat156 + stat157 + stat158 + 
##     stat159 + stat160 + stat161 + stat162 + stat163 + stat164 + 
##     stat165 + stat166 + stat167 + stat168 + stat169 + stat170 + 
##     stat171 + stat172 + stat173 + stat174 + stat175 + stat176 + 
##     stat177 + stat178 + stat179 + stat180 + stat181 + stat182 + 
##     stat183 + stat184 + stat185 + stat186 + stat187 + stat188 + 
##     stat189 + stat190 + stat191 + stat192 + stat193 + stat194 + 
##     stat195 + stat196 + stat197 + stat198 + stat199 + stat200 + 
##     stat201 + stat202 + stat203 + stat204 + stat205 + stat206 + 
##     stat207 + stat208 + stat209 + stat210 + stat211 + stat212 + 
##     stat213 + stat214 + stat215 + stat216 + stat217 + x18.sqrt
print(grand.mean.formula)
## y3.log ~ 1
# Update feature.names because we may have transformed some features
feature.names = n[!n %in% label.names]

Full Model

model.full = lm(formula , data.train)
summary(model.full)
## 
## Call:
## lm(formula = formula, data = data.train)
## 
## Residuals:
##       Min        1Q    Median        3Q       Max 
## -0.083002 -0.020595 -0.004662  0.015985  0.192768 
## 
## Coefficients:
##               Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  1.974e+00  9.614e-03 205.338  < 2e-16 ***
## x1           9.057e-05  6.572e-04   0.138 0.890404    
## x2           1.063e-04  4.220e-04   0.252 0.801208    
## x3           9.585e-05  1.160e-04   0.826 0.408562    
## x4          -4.421e-05  9.122e-06  -4.846 1.29e-06 ***
## x5           2.621e-04  3.005e-04   0.872 0.383064    
## x6           3.306e-04  6.016e-04   0.549 0.582689    
## x7           1.136e-02  6.414e-04  17.711  < 2e-16 ***
## x8           4.468e-04  1.502e-04   2.975 0.002944 ** 
## x9           3.152e-03  3.361e-04   9.379  < 2e-16 ***
## x10          1.102e-03  3.102e-04   3.553 0.000384 ***
## x11          1.298e+05  7.442e+04   1.744 0.081181 .  
## x12         -1.890e-04  1.903e-04  -0.993 0.320637    
## x13          2.529e-06  7.574e-05   0.033 0.973368    
## x14         -6.223e-04  3.273e-04  -1.901 0.057301 .  
## x15          2.747e-04  3.108e-04   0.884 0.376793    
## x16          9.860e-04  2.149e-04   4.588 4.57e-06 ***
## x17          1.749e-03  3.290e-04   5.315 1.11e-07 ***
## x19          1.545e-04  1.675e-04   0.923 0.356193    
## x20         -8.534e-04  1.150e-03  -0.742 0.458154    
## x21          1.577e-04  4.263e-05   3.701 0.000217 ***
## x22         -3.667e-04  3.484e-04  -1.053 0.292531    
## x23         -5.354e-05  3.308e-04  -0.162 0.871439    
## stat1       -1.341e-04  2.502e-04  -0.536 0.592021    
## stat2        6.548e-05  2.481e-04   0.264 0.791829    
## stat3        5.315e-04  2.501e-04   2.126 0.033581 *  
## stat4       -4.065e-04  2.510e-04  -1.619 0.105488    
## stat5        5.502e-05  2.504e-04   0.220 0.826074    
## stat6       -2.524e-04  2.508e-04  -1.007 0.314167    
## stat7       -3.159e-04  2.521e-04  -1.253 0.210260    
## stat8        3.549e-04  2.500e-04   1.419 0.155838    
## stat9        1.920e-04  2.502e-04   0.767 0.442965    
## stat10      -2.410e-04  2.503e-04  -0.963 0.335646    
## stat11      -2.545e-04  2.517e-04  -1.011 0.312024    
## stat12       2.624e-04  2.501e-04   1.049 0.294110    
## stat13      -7.778e-04  2.486e-04  -3.129 0.001765 ** 
## stat14      -9.028e-04  2.487e-04  -3.631 0.000285 ***
## stat15      -1.941e-04  2.492e-04  -0.779 0.436230    
## stat16       9.641e-05  2.493e-04   0.387 0.698968    
## stat17       2.100e-04  2.476e-04   0.848 0.396377    
## stat18      -3.954e-04  2.495e-04  -1.584 0.113159    
## stat19       2.505e-04  2.487e-04   1.007 0.313812    
## stat20      -4.358e-04  2.501e-04  -1.743 0.081465 .  
## stat21       2.223e-04  2.492e-04   0.892 0.372460    
## stat22      -4.646e-04  2.499e-04  -1.859 0.063047 .  
## stat23       7.015e-04  2.490e-04   2.817 0.004862 ** 
## stat24      -7.599e-04  2.505e-04  -3.034 0.002424 ** 
## stat25      -3.405e-04  2.496e-04  -1.364 0.172535    
## stat26      -3.245e-04  2.517e-04  -1.289 0.197323    
## stat27       8.270e-06  2.504e-04   0.033 0.973653    
## stat28       7.367e-05  2.498e-04   0.295 0.768096    
## stat29       1.392e-04  2.521e-04   0.552 0.581014    
## stat30       2.403e-04  2.526e-04   0.951 0.341560    
## stat31      -2.519e-04  2.529e-04  -0.996 0.319259    
## stat32       6.420e-05  2.518e-04   0.255 0.798788    
## stat33      -3.200e-04  2.496e-04  -1.282 0.199811    
## stat34      -5.998e-05  2.493e-04  -0.241 0.809919    
## stat35      -1.631e-04  2.496e-04  -0.653 0.513642    
## stat36      -8.096e-05  2.494e-04  -0.325 0.745482    
## stat37      -3.361e-04  2.507e-04  -1.341 0.180099    
## stat38       5.130e-04  2.521e-04   2.035 0.041899 *  
## stat39      -2.774e-04  2.494e-04  -1.112 0.266014    
## stat40      -7.034e-05  2.512e-04  -0.280 0.779511    
## stat41      -3.730e-04  2.469e-04  -1.511 0.130876    
## stat42      -3.845e-04  2.509e-04  -1.532 0.125502    
## stat43      -1.308e-04  2.510e-04  -0.521 0.602302    
## stat44       3.056e-04  2.509e-04   1.218 0.223238    
## stat45      -3.370e-04  2.496e-04  -1.350 0.176969    
## stat46       3.710e-04  2.501e-04   1.483 0.138065    
## stat47       7.464e-05  2.526e-04   0.296 0.767619    
## stat48       5.688e-05  2.511e-04   0.227 0.820790    
## stat49       3.486e-04  2.479e-04   1.406 0.159755    
## stat50       1.134e-04  2.477e-04   0.458 0.647198    
## stat51       4.929e-04  2.506e-04   1.967 0.049292 *  
## stat52      -9.487e-05  2.505e-04  -0.379 0.704881    
## stat53      -2.501e-04  2.530e-04  -0.989 0.322889    
## stat54      -3.201e-04  2.520e-04  -1.270 0.204044    
## stat55       3.067e-04  2.474e-04   1.240 0.215183    
## stat56       4.815e-05  2.523e-04   0.191 0.848662    
## stat57       2.240e-04  2.473e-04   0.906 0.365211    
## stat58       1.208e-05  2.483e-04   0.049 0.961206    
## stat59       2.999e-04  2.505e-04   1.197 0.231185    
## stat60       6.698e-04  2.508e-04   2.671 0.007593 ** 
## stat61      -4.609e-05  2.523e-04  -0.183 0.855044    
## stat62      -1.232e-04  2.488e-04  -0.495 0.620340    
## stat63       1.550e-06  2.502e-04   0.006 0.995058    
## stat64      -7.644e-05  2.497e-04  -0.306 0.759527    
## stat65      -4.909e-04  2.512e-04  -1.954 0.050710 .  
## stat66       1.822e-04  2.523e-04   0.722 0.470248    
## stat67       6.368e-05  2.510e-04   0.254 0.799718    
## stat68      -7.074e-05  2.511e-04  -0.282 0.778182    
## stat69       1.552e-04  2.511e-04   0.618 0.536593    
## stat70       2.284e-04  2.490e-04   0.917 0.359101    
## stat71       1.543e-04  2.492e-04   0.619 0.535810    
## stat72      -1.673e-05  2.529e-04  -0.066 0.947247    
## stat73       3.211e-04  2.515e-04   1.277 0.201760    
## stat74      -2.061e-04  2.508e-04  -0.822 0.411235    
## stat75      -1.580e-04  2.522e-04  -0.627 0.530984    
## stat76       2.166e-04  2.513e-04   0.862 0.388613    
## stat77       8.697e-05  2.502e-04   0.348 0.728169    
## stat78       3.983e-06  2.498e-04   0.016 0.987280    
## stat79      -2.369e-04  2.515e-04  -0.942 0.346300    
## stat80       6.062e-05  2.510e-04   0.242 0.809144    
## stat81       2.465e-04  2.530e-04   0.974 0.329868    
## stat82       1.916e-04  2.503e-04   0.766 0.443962    
## stat83       7.248e-06  2.502e-04   0.029 0.976889    
## stat84      -3.088e-04  2.487e-04  -1.241 0.214546    
## stat85       1.119e-04  2.509e-04   0.446 0.655566    
## stat86       6.028e-06  2.496e-04   0.024 0.980730    
## stat87      -2.929e-04  2.509e-04  -1.167 0.243200    
## stat88      -2.253e-04  2.466e-04  -0.913 0.361060    
## stat89      -2.327e-04  2.489e-04  -0.935 0.349772    
## stat90      -2.982e-04  2.513e-04  -1.187 0.235325    
## stat91      -5.657e-04  2.488e-04  -2.274 0.023033 *  
## stat92      -1.412e-04  2.502e-04  -0.564 0.572628    
## stat93      -4.721e-05  2.513e-04  -0.188 0.851005    
## stat94      -2.079e-04  2.510e-04  -0.828 0.407607    
## stat95       1.997e-05  2.516e-04   0.079 0.936741    
## stat96      -2.710e-04  2.493e-04  -1.087 0.277096    
## stat97       1.084e-05  2.482e-04   0.044 0.965156    
## stat98       3.578e-03  2.470e-04  14.483  < 2e-16 ***
## stat99       4.987e-04  2.512e-04   1.985 0.047189 *  
## stat100      5.600e-04  2.494e-04   2.245 0.024789 *  
## stat101     -2.431e-04  2.513e-04  -0.967 0.333455    
## stat102      9.486e-05  2.515e-04   0.377 0.706067    
## stat103     -5.426e-04  2.525e-04  -2.149 0.031687 *  
## stat104     -3.374e-04  2.503e-04  -1.348 0.177775    
## stat105     -3.040e-05  2.489e-04  -0.122 0.902782    
## stat106     -2.777e-04  2.501e-04  -1.110 0.266970    
## stat107     -1.363e-04  2.480e-04  -0.550 0.582683    
## stat108     -3.184e-04  2.489e-04  -1.279 0.200879    
## stat109      8.727e-05  2.493e-04   0.350 0.726331    
## stat110     -3.512e-03  2.494e-04 -14.084  < 2e-16 ***
## stat111      1.120e-05  2.500e-04   0.045 0.964263    
## stat112     -1.434e-04  2.516e-04  -0.570 0.568631    
## stat113     -1.488e-04  2.518e-04  -0.591 0.554547    
## stat114      7.453e-05  2.515e-04   0.296 0.767002    
## stat115      3.912e-04  2.501e-04   1.564 0.117823    
## stat116      1.875e-04  2.514e-04   0.746 0.455899    
## stat117      3.483e-05  2.520e-04   0.138 0.890064    
## stat118     -1.382e-04  2.499e-04  -0.553 0.580245    
## stat119      1.304e-04  2.506e-04   0.520 0.602847    
## stat120      2.200e-04  2.499e-04   0.880 0.378722    
## stat121     -1.369e-04  2.512e-04  -0.545 0.585847    
## stat122     -2.813e-05  2.486e-04  -0.113 0.909922    
## stat123      2.596e-05  2.540e-04   0.102 0.918582    
## stat124      4.223e-07  2.502e-04   0.002 0.998654    
## stat125     -3.122e-05  2.511e-04  -0.124 0.901079    
## stat126      1.488e-04  2.506e-04   0.594 0.552813    
## stat127      1.203e-04  2.504e-04   0.481 0.630795    
## stat128     -2.465e-04  2.487e-04  -0.991 0.321637    
## stat129      6.779e-05  2.491e-04   0.272 0.785483    
## stat130      2.912e-04  2.508e-04   1.161 0.245665    
## stat131      2.240e-04  2.510e-04   0.892 0.372294    
## stat132      2.278e-04  2.490e-04   0.915 0.360248    
## stat133      4.173e-05  2.511e-04   0.166 0.868005    
## stat134     -4.537e-04  2.485e-04  -1.826 0.067942 .  
## stat135     -3.553e-06  2.493e-04  -0.014 0.988630    
## stat136     -2.152e-04  2.508e-04  -0.858 0.390976    
## stat137     -2.436e-06  2.486e-04  -0.010 0.992181    
## stat138      1.518e-04  2.495e-04   0.608 0.542890    
## stat139      1.702e-04  2.511e-04   0.678 0.497749    
## stat140      1.874e-04  2.498e-04   0.750 0.453170    
## stat141     -6.087e-05  2.479e-04  -0.246 0.806059    
## stat142     -1.568e-04  2.518e-04  -0.623 0.533501    
## stat143      1.169e-04  2.501e-04   0.467 0.640220    
## stat144      4.554e-04  2.498e-04   1.823 0.068355 .  
## stat145     -1.093e-04  2.534e-04  -0.432 0.666095    
## stat146     -5.217e-04  2.530e-04  -2.062 0.039284 *  
## stat147     -1.809e-04  2.518e-04  -0.718 0.472592    
## stat148     -3.597e-04  2.470e-04  -1.456 0.145399    
## stat149     -6.940e-04  2.524e-04  -2.750 0.005978 ** 
## stat150     -1.627e-04  2.524e-04  -0.645 0.519174    
## stat151      2.486e-05  2.528e-04   0.098 0.921672    
## stat152     -9.891e-05  2.498e-04  -0.396 0.692133    
## stat153     -8.082e-05  2.530e-04  -0.319 0.749444    
## stat154      1.827e-04  2.534e-04   0.721 0.470980    
## stat155     -1.410e-04  2.481e-04  -0.568 0.569941    
## stat156      4.656e-04  2.521e-04   1.847 0.064772 .  
## stat157     -4.236e-05  2.486e-04  -0.170 0.864681    
## stat158     -1.930e-04  2.546e-04  -0.758 0.448560    
## stat159      1.660e-05  2.495e-04   0.067 0.946970    
## stat160     -2.578e-05  2.508e-04  -0.103 0.918123    
## stat161      2.016e-04  2.510e-04   0.803 0.421889    
## stat162      1.277e-04  2.493e-04   0.512 0.608479    
## stat163      1.270e-04  2.535e-04   0.501 0.616424    
## stat164      8.564e-05  2.526e-04   0.339 0.734638    
## stat165     -2.348e-04  2.468e-04  -0.952 0.341330    
## stat166     -2.764e-04  2.478e-04  -1.115 0.264713    
## stat167     -2.959e-04  2.507e-04  -1.180 0.237899    
## stat168     -1.918e-04  2.492e-04  -0.770 0.441612    
## stat169      2.133e-05  2.502e-04   0.085 0.932061    
## stat170     -4.355e-04  2.507e-04  -1.737 0.082377 .  
## stat171      3.402e-05  2.514e-04   0.135 0.892374    
## stat172      1.647e-04  2.476e-04   0.665 0.506085    
## stat173     -9.253e-05  2.519e-04  -0.367 0.713424    
## stat174     -2.616e-06  2.498e-04  -0.010 0.991646    
## stat175     -3.947e-04  2.512e-04  -1.571 0.116285    
## stat176      1.321e-04  2.505e-04   0.527 0.597931    
## stat177     -5.291e-05  2.512e-04  -0.211 0.833161    
## stat178     -2.908e-04  2.536e-04  -1.147 0.251604    
## stat179      1.258e-04  2.499e-04   0.504 0.614606    
## stat180     -2.460e-04  2.476e-04  -0.994 0.320490    
## stat181      2.603e-04  2.514e-04   1.035 0.300582    
## stat182      2.010e-04  2.505e-04   0.802 0.422398    
## stat183      1.557e-04  2.502e-04   0.622 0.533671    
## stat184      1.249e-05  2.515e-04   0.050 0.960399    
## stat185     -2.267e-04  2.477e-04  -0.915 0.360024    
## stat186     -1.816e-04  2.526e-04  -0.719 0.472287    
## stat187     -5.476e-04  2.497e-04  -2.193 0.028332 *  
## stat188     -1.335e-04  2.488e-04  -0.537 0.591609    
## stat189      1.979e-04  2.504e-04   0.790 0.429366    
## stat190      1.013e-04  2.494e-04   0.406 0.684531    
## stat191     -2.085e-04  2.496e-04  -0.835 0.403509    
## stat192     -1.939e-04  2.524e-04  -0.768 0.442394    
## stat193     -9.326e-05  2.525e-04  -0.369 0.711908    
## stat194     -5.444e-05  2.484e-04  -0.219 0.826548    
## stat195      6.156e-04  2.487e-04   2.476 0.013335 *  
## stat196     -2.075e-04  2.543e-04  -0.816 0.414502    
## stat197      8.679e-05  2.482e-04   0.350 0.726587    
## stat198     -3.840e-04  2.499e-04  -1.537 0.124452    
## stat199      9.830e-05  2.491e-04   0.395 0.693185    
## stat200     -2.675e-04  2.466e-04  -1.085 0.277933    
## stat201      8.783e-05  2.490e-04   0.353 0.724238    
## stat202     -3.000e-04  2.541e-04  -1.181 0.237730    
## stat203      3.950e-06  2.498e-04   0.016 0.987388    
## stat204     -4.336e-04  2.488e-04  -1.742 0.081493 .  
## stat205     -2.211e-04  2.499e-04  -0.885 0.376314    
## stat206      5.090e-05  2.519e-04   0.202 0.839876    
## stat207      3.264e-04  2.494e-04   1.309 0.190662    
## stat208      5.885e-05  2.522e-04   0.233 0.815546    
## stat209      1.293e-05  2.492e-04   0.052 0.958606    
## stat210     -2.276e-05  2.504e-04  -0.091 0.927594    
## stat211      3.833e-05  2.511e-04   0.153 0.878666    
## stat212     -4.135e-05  2.510e-04  -0.165 0.869161    
## stat213     -3.530e-04  2.516e-04  -1.403 0.160751    
## stat214     -1.016e-04  2.508e-04  -0.405 0.685394    
## stat215     -1.351e-04  2.499e-04  -0.541 0.588798    
## stat216     -2.761e-04  2.499e-04  -1.105 0.269285    
## stat217      2.641e-04  2.497e-04   1.057 0.290344    
## x18.sqrt     2.615e-02  9.579e-04  27.304  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.03168 on 5343 degrees of freedom
## Multiple R-squared:  0.2708, Adjusted R-squared:  0.238 
## F-statistic: 8.266 on 240 and 5343 DF,  p-value: < 2.2e-16
cd.full = plot.diagnostics(model=model.full, train=data.train)

## [1] "Number of data points that have Cook's D > 4/n: 276"
## [1] "Number of data points that have Cook's D > 1: 0"

Checking with removal of high influence points

high.cd = names(cd.full[cd.full > 4/nrow(data.train)])
data.train2 = data.train[!(rownames(data.train)) %in% high.cd,]
model.full2 = lm(formula , data.train2)
summary(model.full2)
## 
## Call:
## lm(formula = formula, data = data.train2)
## 
## Residuals:
##       Min        1Q    Median        3Q       Max 
## -0.058403 -0.017567 -0.002509  0.016246  0.071465 
## 
## Coefficients:
##               Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  1.964e+00  7.845e-03 250.375  < 2e-16 ***
## x1           3.320e-05  5.375e-04   0.062 0.950740    
## x2           1.367e-04  3.444e-04   0.397 0.691501    
## x3           9.565e-05  9.441e-05   1.013 0.311014    
## x4          -4.713e-05  7.460e-06  -6.317 2.89e-10 ***
## x5           4.835e-04  2.452e-04   1.972 0.048712 *  
## x6          -2.217e-04  4.910e-04  -0.452 0.651619    
## x7           1.226e-02  5.241e-04  23.396  < 2e-16 ***
## x8           5.206e-04  1.227e-04   4.243 2.25e-05 ***
## x9           3.033e-03  2.736e-04  11.084  < 2e-16 ***
## x10          1.536e-03  2.538e-04   6.050 1.56e-09 ***
## x11          1.659e+05  6.090e+04   2.724 0.006464 ** 
## x12         -5.622e-05  1.548e-04  -0.363 0.716476    
## x13          5.657e-05  6.197e-05   0.913 0.361393    
## x14         -4.776e-04  2.669e-04  -1.789 0.073610 .  
## x15          1.361e-04  2.541e-04   0.536 0.592277    
## x16          9.840e-04  1.753e-04   5.612 2.11e-08 ***
## x17          1.759e-03  2.689e-04   6.542 6.67e-11 ***
## x19         -6.173e-06  1.369e-04  -0.045 0.964040    
## x20         -1.017e-03  9.395e-04  -1.083 0.278881    
## x21          1.481e-04  3.481e-05   4.255 2.13e-05 ***
## x22         -5.310e-04  2.841e-04  -1.869 0.061649 .  
## x23          3.744e-05  2.703e-04   0.139 0.889841    
## stat1       -1.325e-04  2.039e-04  -0.650 0.516033    
## stat2        1.848e-04  2.023e-04   0.913 0.361066    
## stat3        5.713e-04  2.043e-04   2.796 0.005193 ** 
## stat4       -4.609e-04  2.054e-04  -2.244 0.024890 *  
## stat5       -3.637e-05  2.050e-04  -0.177 0.859165    
## stat6       -3.124e-04  2.047e-04  -1.526 0.127042    
## stat7       -3.300e-04  2.055e-04  -1.606 0.108314    
## stat8        2.993e-04  2.040e-04   1.467 0.142418    
## stat9        2.583e-05  2.050e-04   0.126 0.899744    
## stat10      -2.834e-04  2.040e-04  -1.390 0.164703    
## stat11      -3.168e-04  2.058e-04  -1.540 0.123711    
## stat12       2.079e-04  2.039e-04   1.019 0.308075    
## stat13      -7.533e-04  2.030e-04  -3.710 0.000209 ***
## stat14      -9.968e-04  2.031e-04  -4.908 9.50e-07 ***
## stat15      -3.570e-04  2.038e-04  -1.752 0.079884 .  
## stat16      -9.297e-05  2.030e-04  -0.458 0.646929    
## stat17       1.649e-04  2.027e-04   0.814 0.415920    
## stat18      -3.918e-04  2.036e-04  -1.925 0.054345 .  
## stat19       2.743e-04  2.038e-04   1.346 0.178480    
## stat20       1.853e-05  2.042e-04   0.091 0.927709    
## stat21       6.695e-05  2.033e-04   0.329 0.741853    
## stat22      -2.688e-04  2.037e-04  -1.319 0.187071    
## stat23       5.222e-04  2.039e-04   2.561 0.010463 *  
## stat24      -7.672e-04  2.047e-04  -3.748 0.000180 ***
## stat25      -1.866e-04  2.038e-04  -0.915 0.360074    
## stat26      -3.968e-04  2.057e-04  -1.929 0.053772 .  
## stat27      -8.368e-05  2.052e-04  -0.408 0.683443    
## stat28      -5.458e-05  2.044e-04  -0.267 0.789487    
## stat29       1.444e-04  2.058e-04   0.702 0.482851    
## stat30       1.895e-04  2.060e-04   0.920 0.357569    
## stat31      -1.435e-04  2.065e-04  -0.695 0.487038    
## stat32       5.130e-05  2.056e-04   0.249 0.803032    
## stat33      -2.928e-04  2.039e-04  -1.436 0.151033    
## stat34       5.665e-05  2.035e-04   0.278 0.780745    
## stat35      -3.277e-04  2.040e-04  -1.607 0.108171    
## stat36      -2.260e-06  2.037e-04  -0.011 0.991149    
## stat37      -1.127e-04  2.049e-04  -0.550 0.582512    
## stat38       6.346e-04  2.056e-04   3.087 0.002030 ** 
## stat39      -2.685e-04  2.033e-04  -1.320 0.186748    
## stat40      -6.101e-05  2.057e-04  -0.297 0.766811    
## stat41      -4.129e-04  2.014e-04  -2.051 0.040365 *  
## stat42      -2.172e-04  2.052e-04  -1.059 0.289780    
## stat43      -1.055e-04  2.049e-04  -0.515 0.606549    
## stat44       4.275e-04  2.051e-04   2.084 0.037205 *  
## stat45      -1.516e-04  2.037e-04  -0.744 0.456749    
## stat46       1.005e-04  2.042e-04   0.492 0.622461    
## stat47       2.086e-04  2.057e-04   1.014 0.310542    
## stat48       8.389e-05  2.045e-04   0.410 0.681636    
## stat49       9.420e-05  2.026e-04   0.465 0.642007    
## stat50       1.670e-04  2.025e-04   0.825 0.409467    
## stat51       3.295e-04  2.046e-04   1.610 0.107396    
## stat52       3.872e-05  2.048e-04   0.189 0.850012    
## stat53      -2.525e-04  2.068e-04  -1.221 0.222172    
## stat54      -2.355e-04  2.062e-04  -1.142 0.253385    
## stat55       1.421e-04  2.021e-04   0.703 0.482199    
## stat56       2.066e-04  2.060e-04   1.003 0.315963    
## stat57       1.077e-04  2.024e-04   0.532 0.594610    
## stat58      -8.317e-05  2.025e-04  -0.411 0.681234    
## stat59       2.702e-04  2.044e-04   1.322 0.186209    
## stat60       6.751e-04  2.048e-04   3.297 0.000985 ***
## stat61      -2.050e-04  2.060e-04  -0.995 0.319779    
## stat62      -2.294e-04  2.029e-04  -1.131 0.258151    
## stat63      -9.947e-05  2.045e-04  -0.486 0.626773    
## stat64      -1.621e-05  2.035e-04  -0.080 0.936536    
## stat65      -1.730e-04  2.051e-04  -0.844 0.398948    
## stat66       8.833e-05  2.060e-04   0.429 0.668093    
## stat67       2.250e-04  2.048e-04   1.098 0.272111    
## stat68      -1.048e-04  2.050e-04  -0.511 0.609025    
## stat69      -1.656e-05  2.052e-04  -0.081 0.935687    
## stat70       2.890e-04  2.033e-04   1.421 0.155291    
## stat71       2.014e-04  2.037e-04   0.989 0.322855    
## stat72      -2.314e-04  2.064e-04  -1.121 0.262440    
## stat73       3.108e-04  2.056e-04   1.512 0.130600    
## stat74      -9.969e-05  2.049e-04  -0.487 0.626625    
## stat75       4.215e-05  2.060e-04   0.205 0.837882    
## stat76       2.277e-04  2.051e-04   1.110 0.266913    
## stat77       3.226e-04  2.044e-04   1.578 0.114538    
## stat78      -2.161e-04  2.033e-04  -1.063 0.288004    
## stat79      -1.237e-04  2.050e-04  -0.603 0.546488    
## stat80       1.120e-04  2.048e-04   0.547 0.584604    
## stat81       1.601e-04  2.068e-04   0.774 0.438793    
## stat82       1.001e-04  2.044e-04   0.490 0.624161    
## stat83       7.159e-05  2.043e-04   0.350 0.726086    
## stat84      -3.448e-04  2.031e-04  -1.698 0.089558 .  
## stat85      -3.260e-04  2.048e-04  -1.592 0.111438    
## stat86       1.919e-04  2.042e-04   0.940 0.347274    
## stat87      -3.755e-04  2.049e-04  -1.832 0.066962 .  
## stat88      -1.393e-04  2.017e-04  -0.691 0.489878    
## stat89      -9.282e-05  2.038e-04  -0.455 0.648885    
## stat90      -3.094e-04  2.051e-04  -1.508 0.131574    
## stat91      -5.718e-04  2.027e-04  -2.821 0.004810 ** 
## stat92       1.517e-06  2.041e-04   0.007 0.994069    
## stat93       2.039e-05  2.059e-04   0.099 0.921128    
## stat94       1.076e-04  2.046e-04   0.526 0.598921    
## stat95       8.862e-05  2.058e-04   0.431 0.666782    
## stat96      -2.130e-04  2.039e-04  -1.045 0.296248    
## stat97       1.228e-04  2.025e-04   0.606 0.544319    
## stat98       3.345e-03  2.016e-04  16.595  < 2e-16 ***
## stat99       4.072e-04  2.053e-04   1.984 0.047347 *  
## stat100      7.281e-04  2.035e-04   3.578 0.000350 ***
## stat101     -5.506e-05  2.054e-04  -0.268 0.788603    
## stat102      2.029e-04  2.053e-04   0.988 0.323269    
## stat103     -5.420e-04  2.059e-04  -2.633 0.008497 ** 
## stat104     -2.152e-04  2.049e-04  -1.050 0.293599    
## stat105      9.886e-05  2.031e-04   0.487 0.626443    
## stat106     -2.829e-04  2.042e-04  -1.386 0.165855    
## stat107     -4.746e-06  2.024e-04  -0.023 0.981292    
## stat108     -2.479e-04  2.037e-04  -1.217 0.223743    
## stat109      7.569e-05  2.037e-04   0.372 0.710275    
## stat110     -3.431e-03  2.036e-04 -16.852  < 2e-16 ***
## stat111     -8.182e-05  2.041e-04  -0.401 0.688498    
## stat112     -1.403e-04  2.058e-04  -0.682 0.495519    
## stat113     -9.204e-05  2.055e-04  -0.448 0.654317    
## stat114      1.015e-04  2.057e-04   0.493 0.621851    
## stat115      4.160e-04  2.045e-04   2.034 0.041998 *  
## stat116      2.372e-04  2.054e-04   1.155 0.248267    
## stat117      3.634e-05  2.053e-04   0.177 0.859521    
## stat118      1.199e-04  2.040e-04   0.588 0.556805    
## stat119      2.841e-05  2.050e-04   0.139 0.889781    
## stat120     -2.410e-05  2.039e-04  -0.118 0.905921    
## stat121     -1.188e-04  2.052e-04  -0.579 0.562569    
## stat122     -2.328e-04  2.034e-04  -1.144 0.252542    
## stat123      2.032e-04  2.070e-04   0.982 0.326129    
## stat124      2.225e-05  2.042e-04   0.109 0.913221    
## stat125     -1.922e-04  2.052e-04  -0.937 0.349022    
## stat126      7.221e-05  2.046e-04   0.353 0.724134    
## stat127      1.342e-05  2.040e-04   0.066 0.947558    
## stat128     -4.508e-04  2.027e-04  -2.224 0.026218 *  
## stat129      8.227e-05  2.036e-04   0.404 0.686134    
## stat130      3.254e-04  2.048e-04   1.589 0.112227    
## stat131      1.221e-04  2.047e-04   0.596 0.550940    
## stat132      1.574e-04  2.034e-04   0.774 0.439151    
## stat133      1.603e-04  2.052e-04   0.781 0.434836    
## stat134     -4.073e-04  2.027e-04  -2.010 0.044522 *  
## stat135     -1.531e-04  2.035e-04  -0.753 0.451651    
## stat136     -3.051e-04  2.046e-04  -1.491 0.135934    
## stat137      5.909e-05  2.030e-04   0.291 0.770936    
## stat138      4.380e-05  2.037e-04   0.215 0.829743    
## stat139      5.778e-05  2.054e-04   0.281 0.778454    
## stat140      1.933e-04  2.030e-04   0.952 0.341008    
## stat141      1.432e-04  2.024e-04   0.708 0.479108    
## stat142     -1.305e-04  2.057e-04  -0.634 0.525792    
## stat143     -4.836e-05  2.047e-04  -0.236 0.813218    
## stat144      5.469e-04  2.039e-04   2.682 0.007344 ** 
## stat145     -1.977e-04  2.070e-04  -0.955 0.339671    
## stat146     -5.518e-04  2.065e-04  -2.671 0.007576 ** 
## stat147     -1.999e-04  2.056e-04  -0.972 0.331035    
## stat148     -2.721e-04  2.021e-04  -1.347 0.178190    
## stat149     -6.869e-04  2.067e-04  -3.323 0.000896 ***
## stat150     -2.878e-04  2.069e-04  -1.391 0.164279    
## stat151      3.584e-04  2.067e-04   1.734 0.083064 .  
## stat152     -1.478e-04  2.039e-04  -0.725 0.468606    
## stat153      1.403e-04  2.062e-04   0.680 0.496488    
## stat154      3.410e-04  2.074e-04   1.644 0.100181    
## stat155      1.329e-04  2.027e-04   0.656 0.512093    
## stat156      3.243e-04  2.058e-04   1.576 0.115162    
## stat157      6.913e-06  2.030e-04   0.034 0.972832    
## stat158     -1.366e-05  2.078e-04  -0.066 0.947618    
## stat159     -2.042e-05  2.036e-04  -0.100 0.920097    
## stat160     -7.693e-05  2.049e-04  -0.375 0.707324    
## stat161      5.339e-05  2.050e-04   0.260 0.794570    
## stat162      8.464e-06  2.029e-04   0.042 0.966725    
## stat163      2.652e-04  2.078e-04   1.276 0.201977    
## stat164     -2.987e-05  2.068e-04  -0.144 0.885125    
## stat165      1.239e-05  2.018e-04   0.061 0.951046    
## stat166     -2.622e-04  2.021e-04  -1.297 0.194644    
## stat167     -2.680e-04  2.047e-04  -1.309 0.190501    
## stat168     -2.041e-04  2.033e-04  -1.004 0.315656    
## stat169      1.856e-05  2.045e-04   0.091 0.927692    
## stat170     -3.008e-04  2.047e-04  -1.470 0.141632    
## stat171     -1.603e-04  2.054e-04  -0.780 0.435158    
## stat172      3.703e-04  2.020e-04   1.834 0.066785 .  
## stat173      5.051e-05  2.053e-04   0.246 0.805628    
## stat174      1.828e-04  2.040e-04   0.896 0.370268    
## stat175     -3.380e-04  2.048e-04  -1.650 0.098923 .  
## stat176     -4.700e-05  2.048e-04  -0.230 0.818476    
## stat177     -3.897e-04  2.050e-04  -1.901 0.057326 .  
## stat178     -7.513e-05  2.069e-04  -0.363 0.716590    
## stat179      2.926e-05  2.038e-04   0.144 0.885873    
## stat180     -9.601e-05  2.027e-04  -0.474 0.635718    
## stat181      3.952e-04  2.050e-04   1.927 0.053988 .  
## stat182      4.176e-04  2.047e-04   2.040 0.041368 *  
## stat183      1.883e-04  2.047e-04   0.920 0.357564    
## stat184      2.480e-04  2.054e-04   1.208 0.227159    
## stat185      1.972e-05  2.025e-04   0.097 0.922424    
## stat186      1.179e-04  2.060e-04   0.572 0.567061    
## stat187     -4.184e-04  2.038e-04  -2.053 0.040149 *  
## stat188      1.183e-04  2.032e-04   0.582 0.560357    
## stat189     -1.109e-05  2.048e-04  -0.054 0.956796    
## stat190     -1.316e-04  2.037e-04  -0.646 0.518293    
## stat191     -1.127e-04  2.036e-04  -0.554 0.579916    
## stat192     -1.689e-04  2.063e-04  -0.819 0.412999    
## stat193     -9.613e-05  2.065e-04  -0.466 0.641541    
## stat194     -2.763e-04  2.032e-04  -1.360 0.173950    
## stat195      2.819e-04  2.030e-04   1.388 0.165139    
## stat196     -2.245e-04  2.076e-04  -1.081 0.279627    
## stat197     -8.532e-05  2.028e-04  -0.421 0.674016    
## stat198     -3.000e-04  2.038e-04  -1.472 0.141198    
## stat199      1.323e-04  2.032e-04   0.651 0.515095    
## stat200     -1.026e-04  2.020e-04  -0.508 0.611570    
## stat201      1.900e-04  2.036e-04   0.933 0.350723    
## stat202     -8.986e-05  2.074e-04  -0.433 0.664795    
## stat203      3.873e-05  2.041e-04   0.190 0.849491    
## stat204     -1.687e-04  2.036e-04  -0.829 0.407227    
## stat205      1.861e-04  2.035e-04   0.915 0.360472    
## stat206     -1.422e-04  2.057e-04  -0.691 0.489437    
## stat207      4.707e-04  2.040e-04   2.307 0.021092 *  
## stat208      1.201e-04  2.059e-04   0.583 0.559905    
## stat209      6.509e-05  2.032e-04   0.320 0.748766    
## stat210     -1.042e-04  2.044e-04  -0.510 0.610367    
## stat211      1.598e-05  2.050e-04   0.078 0.937876    
## stat212      6.775e-05  2.050e-04   0.330 0.741037    
## stat213     -3.338e-04  2.053e-04  -1.626 0.104047    
## stat214      3.406e-06  2.050e-04   0.017 0.986750    
## stat215     -1.037e-05  2.044e-04  -0.051 0.959543    
## stat216     -2.354e-04  2.035e-04  -1.157 0.247380    
## stat217      1.209e-04  2.038e-04   0.593 0.553253    
## x18.sqrt     2.573e-02  7.793e-04  33.013  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.02519 on 5067 degrees of freedom
## Multiple R-squared:  0.3721, Adjusted R-squared:  0.3424 
## F-statistic: 12.51 on 240 and 5067 DF,  p-value: < 2.2e-16
cd.full2 = plot.diagnostics(model.full2, data.train2)

## [1] "Number of data points that have Cook's D > 4/n: 281"
## [1] "Number of data points that have Cook's D > 1: 0"
# much more normal residuals than before. 
# Checking to see if distributions are different and if so whcih variables
# High Leverage Plot 
plotData = data.train %>% 
  rownames_to_column() %>%
  mutate(type=ifelse(rowname %in% high.cd,'High','Normal')) %>%
  dplyr::select(type,target=one_of(label.names))

ggplot(data=plotData, aes(x=type,y=target)) +
  geom_boxplot(fill='light blue',outlier.shape=NA) +
  scale_y_continuous(name="Target Variable Values",label=scales::comma_format(accuracy=.1)) +
  theme_light() +
  ggtitle('Distribution of High Leverage Points and Normal  Points')

# 2 sample t-tests

plotData = data.train %>% 
  rownames_to_column() %>%
  mutate(type=ifelse(rowname %in% high.cd,'High','Normal')) %>%
  dplyr::select(type,one_of(feature.names))

comp.test = lapply(dplyr::select(plotData, one_of(feature.names))
                   , function(x) t.test(x ~ plotData$type, var.equal = TRUE)) 

sig.comp = list.filter(comp.test, p.value < 0.05)
sapply(sig.comp, function(x) x[['p.value']])
##           x4       stat38       stat85       stat98      stat110      stat151     x18.sqrt 
## 2.372263e-02 1.980292e-02 4.248720e-03 4.940805e-06 2.415654e-03 3.490989e-02 3.601163e-03
mm = melt(plotData, id=c('type')) %>% filter(variable %in% names(sig.comp))

ggplot(mm,aes(x=type, y=value)) +
  geom_boxplot()+
  facet_wrap(~variable, ncol=5, scales = 'free_y') +
  scale_y_continuous(name="values",label=scales::comma_format(accuracy=.1)) +
  ggtitle('Distribution of High Leverage Points and Normal Points')

# Distribution (box) Plots
mm = melt(plotData, id=c('type'))

ggplot(mm,aes(x=type, y=value)) +
  geom_boxplot()+
  facet_wrap(~variable, ncol=8, scales = 'free_y') +
  scale_y_continuous(name="values",label=scales::comma_format(accuracy=.1)) +
  ggtitle('Distribution of High Leverage Points and Normal Points')

Grand Means Model

model.null = lm(grand.mean.formula, data.train)
summary(model.null)
## 
## Call:
## lm(formula = grand.mean.formula, data = data.train)
## 
## Residuals:
##       Min        1Q    Median        3Q       Max 
## -0.115047 -0.023837 -0.003505  0.020208  0.190265 
## 
## Coefficients:
##              Estimate Std. Error t value Pr(>|t|)    
## (Intercept) 2.0969231  0.0004857    4317   <2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.0363 on 5583 degrees of freedom

Variable Selection

Basic: http://www.stat.columbia.edu/~martin/W2024/R10.pdf Cross Validation + Other Metrics: http://www.sthda.com/english/articles/37-model-selection-essentials-in-r/154-stepwise-regression-essentials-in-r/

Forward Selection with CV

Train

if (algo.forward.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   , data = data.train
                                   , method = "leapForward"
                                   , feature.names = feature.names)
  model.forward = returned$model
  id = returned$id
}
## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 19 on full training set
## [1] "All models results"
##     nvmax       RMSE  Rsquared        MAE      RMSESD RsquaredSD        MAESD
## 1       1 0.03429406 0.1067604 0.02659566 0.001380514 0.01920345 0.0007111804
## 2       2 0.03345922 0.1507056 0.02585671 0.001513502 0.02228332 0.0007636527
## 3       3 0.03305704 0.1708944 0.02540433 0.001593231 0.02013441 0.0008673065
## 4       4 0.03229670 0.2080695 0.02452928 0.001547612 0.01984658 0.0008543763
## 5       5 0.03204362 0.2202654 0.02436176 0.001548044 0.01918191 0.0008650394
## 6       6 0.03204567 0.2200958 0.02436311 0.001523974 0.01873001 0.0008244408
## 7       7 0.03198700 0.2229757 0.02434475 0.001496667 0.01767578 0.0007959974
## 8       8 0.03185614 0.2293116 0.02423250 0.001488678 0.01736199 0.0007882918
## 9       9 0.03189937 0.2272611 0.02425828 0.001483579 0.01744927 0.0007905599
## 10     10 0.03191321 0.2265856 0.02425885 0.001462645 0.01672747 0.0007696399
## 11     11 0.03190249 0.2271851 0.02427821 0.001470544 0.01706492 0.0007674826
## 12     12 0.03188801 0.2279486 0.02427609 0.001476996 0.01703738 0.0007809244
## 13     13 0.03187006 0.2288751 0.02426647 0.001475758 0.01645541 0.0007624986
## 14     14 0.03181575 0.2314520 0.02424750 0.001462587 0.01637929 0.0007558458
## 15     15 0.03182362 0.2311417 0.02424614 0.001477059 0.01817657 0.0007733136
## 16     16 0.03182072 0.2312796 0.02423546 0.001485069 0.01891176 0.0007738498
## 17     17 0.03178287 0.2330225 0.02419076 0.001467648 0.01823470 0.0007503516
## 18     18 0.03177599 0.2333205 0.02418540 0.001474508 0.01864798 0.0007557786
## 19     19 0.03177543 0.2333349 0.02419052 0.001459104 0.01813900 0.0007570142
## 20     20 0.03179310 0.2325367 0.02420905 0.001457838 0.01790560 0.0007669169
## 21     21 0.03181378 0.2316397 0.02422039 0.001455051 0.01784868 0.0007629719
## 22     22 0.03180964 0.2318959 0.02421795 0.001466814 0.01794777 0.0007576435
## 23     23 0.03182812 0.2311100 0.02422467 0.001468962 0.01803978 0.0007574720
## 24     24 0.03184057 0.2305658 0.02423842 0.001477963 0.01825093 0.0007650531
## 25     25 0.03184247 0.2304738 0.02423792 0.001479041 0.01803906 0.0007709289
## 26     26 0.03186990 0.2291860 0.02425438 0.001492675 0.01797547 0.0007745221
## 27     27 0.03187647 0.2289182 0.02426117 0.001487386 0.01806308 0.0007585481
## 28     28 0.03186707 0.2293756 0.02424507 0.001493326 0.01770384 0.0007411117
## 29     29 0.03188183 0.2287443 0.02425450 0.001504187 0.01846859 0.0007602399
## 30     30 0.03189960 0.2279590 0.02426997 0.001493225 0.01762920 0.0007495256
## 31     31 0.03188709 0.2285795 0.02425280 0.001504034 0.01774299 0.0007508989
## 32     32 0.03187933 0.2289594 0.02424559 0.001508522 0.01811995 0.0007565708
## 33     33 0.03188817 0.2285733 0.02424668 0.001526012 0.01860523 0.0007842839
## 34     34 0.03191400 0.2274786 0.02426401 0.001545564 0.01958742 0.0007944432
## 35     35 0.03191148 0.2275748 0.02425919 0.001547989 0.01947719 0.0008006883
## 36     36 0.03190417 0.2279285 0.02425938 0.001554966 0.01999307 0.0008052596
## 37     37 0.03192409 0.2270587 0.02427590 0.001556456 0.02055683 0.0008171430
## 38     38 0.03193970 0.2263360 0.02429002 0.001542032 0.02062746 0.0008076752
## 39     39 0.03196385 0.2252610 0.02430754 0.001539221 0.02065524 0.0008099652
## 40     40 0.03196124 0.2254388 0.02431294 0.001549385 0.02119482 0.0008110985
## 41     41 0.03196934 0.2250862 0.02432172 0.001564956 0.02193039 0.0008223917
## 42     42 0.03196563 0.2253164 0.02432694 0.001570162 0.02230738 0.0008250568
## 43     43 0.03196954 0.2251394 0.02433777 0.001559458 0.02223422 0.0008285842
## 44     44 0.03197266 0.2249826 0.02433658 0.001555213 0.02223006 0.0008273279
## 45     45 0.03198049 0.2246933 0.02433975 0.001554846 0.02221086 0.0008249907
## 46     46 0.03198229 0.2246359 0.02433919 0.001546798 0.02251056 0.0008234678
## 47     47 0.03199286 0.2241983 0.02434865 0.001541125 0.02245215 0.0008189609
## 48     48 0.03199298 0.2241748 0.02434513 0.001528790 0.02153010 0.0008014237
## 49     49 0.03200022 0.2238635 0.02434600 0.001514898 0.02058380 0.0007840816
## 50     50 0.03201379 0.2232627 0.02435703 0.001512467 0.02004060 0.0007856249
## 51     51 0.03201910 0.2230501 0.02436189 0.001515219 0.01951785 0.0007873288
## 52     52 0.03204544 0.2218218 0.02438359 0.001504202 0.01926702 0.0007797856
## 53     53 0.03204362 0.2219459 0.02438621 0.001508182 0.01901184 0.0007720543
## 54     54 0.03202856 0.2226456 0.02436995 0.001502290 0.01869393 0.0007701078
## 55     55 0.03203741 0.2222812 0.02437728 0.001518095 0.01869268 0.0007754109
## 56     56 0.03205513 0.2215240 0.02438690 0.001519784 0.01861012 0.0007740570
## 57     57 0.03206457 0.2211711 0.02438875 0.001515130 0.01852294 0.0007738546
## 58     58 0.03206577 0.2211747 0.02438485 0.001522298 0.01905808 0.0007807620
## 59     59 0.03207445 0.2207884 0.02438909 0.001538679 0.01964435 0.0007836209
## 60     60 0.03207528 0.2207824 0.02439090 0.001546123 0.02010451 0.0007878108
## 61     61 0.03208519 0.2203215 0.02440414 0.001540238 0.01978766 0.0007829013
## 62     62 0.03208651 0.2202389 0.02440323 0.001529709 0.01973918 0.0007677615
## 63     63 0.03208887 0.2201401 0.02441314 0.001528670 0.01951110 0.0007714577
## 64     64 0.03208797 0.2201869 0.02442169 0.001519751 0.01931357 0.0007633310
## 65     65 0.03210027 0.2196808 0.02442549 0.001502904 0.01857059 0.0007461665
## 66     66 0.03211256 0.2191101 0.02442708 0.001501049 0.01862243 0.0007380402
## 67     67 0.03211229 0.2191408 0.02443056 0.001501132 0.01852193 0.0007384116
## 68     68 0.03212238 0.2187313 0.02442933 0.001501596 0.01820153 0.0007345932
## 69     69 0.03213086 0.2183661 0.02442906 0.001499825 0.01821393 0.0007333897
## 70     70 0.03213065 0.2184418 0.02443507 0.001496376 0.01865603 0.0007335201
## 71     71 0.03213766 0.2181251 0.02443693 0.001500223 0.01878195 0.0007399368
## 72     72 0.03213745 0.2181583 0.02444064 0.001489245 0.01829952 0.0007268296
## 73     73 0.03214602 0.2177289 0.02445175 0.001491004 0.01863735 0.0007245366
## 74     74 0.03216508 0.2169306 0.02446706 0.001482750 0.01854746 0.0007196945
## 75     75 0.03217150 0.2166677 0.02447828 0.001486596 0.01860152 0.0007171189
## 76     76 0.03217167 0.2166480 0.02448269 0.001494953 0.01871075 0.0007237273
## 77     77 0.03217468 0.2165584 0.02448744 0.001497078 0.01905971 0.0007246198
## 78     78 0.03218182 0.2162228 0.02449571 0.001486645 0.01858168 0.0007249379
## 79     79 0.03218851 0.2159037 0.02450344 0.001490055 0.01856354 0.0007279638
## 80     80 0.03219424 0.2156466 0.02451182 0.001475413 0.01804534 0.0007147957
## 81     81 0.03220451 0.2152181 0.02451941 0.001479404 0.01828997 0.0007101859
## 82     82 0.03221203 0.2149117 0.02452605 0.001481349 0.01846136 0.0007051348
## 83     83 0.03221650 0.2146922 0.02452420 0.001490890 0.01880358 0.0007181012
## 84     84 0.03223017 0.2141122 0.02453259 0.001480969 0.01832600 0.0007086086
## 85     85 0.03222765 0.2142538 0.02452615 0.001482829 0.01811865 0.0007156790
## 86     86 0.03223138 0.2140941 0.02453494 0.001472560 0.01819553 0.0007083732
## 87     87 0.03224814 0.2133679 0.02454865 0.001459378 0.01773520 0.0006932512
## 88     88 0.03224665 0.2134351 0.02453857 0.001454870 0.01807257 0.0006971341
## 89     89 0.03225604 0.2130738 0.02454396 0.001460732 0.01818110 0.0007001675
## 90     90 0.03226274 0.2128078 0.02454749 0.001462010 0.01813456 0.0007036442
## 91     91 0.03226353 0.2127644 0.02454989 0.001457958 0.01835161 0.0007027062
## 92     92 0.03226041 0.2129233 0.02454482 0.001464713 0.01897549 0.0007056891
## 93     93 0.03225815 0.2130821 0.02453564 0.001476431 0.01971936 0.0007120487
## 94     94 0.03224355 0.2137288 0.02452223 0.001469606 0.01909902 0.0007077825
## 95     95 0.03225254 0.2133541 0.02452342 0.001467562 0.01930108 0.0007031356
## 96     96 0.03224901 0.2135358 0.02452249 0.001469668 0.01948623 0.0007037115
## 97     97 0.03225795 0.2131399 0.02453187 0.001462266 0.01933508 0.0006937872
## 98     98 0.03226368 0.2129173 0.02454121 0.001462870 0.01909877 0.0006917944
## 99     99 0.03226280 0.2129387 0.02454343 0.001465074 0.01870014 0.0006960351
## 100   100 0.03226974 0.2126541 0.02454806 0.001472420 0.01890303 0.0007022552
## 101   101 0.03227055 0.2126583 0.02454844 0.001478474 0.01940265 0.0007082827
## 102   102 0.03228190 0.2121663 0.02456433 0.001469629 0.01926819 0.0007031391
## 103   103 0.03228765 0.2119196 0.02457300 0.001464281 0.01921385 0.0006970594
## 104   104 0.03229434 0.2116625 0.02457651 0.001466320 0.01914586 0.0007012368
## 105   105 0.03229631 0.2115798 0.02457378 0.001450482 0.01882148 0.0006853445
## 106   106 0.03229352 0.2117301 0.02457798 0.001456518 0.01920583 0.0006877640
## 107   107 0.03228808 0.2119661 0.02457630 0.001461295 0.01922991 0.0006977074
## 108   108 0.03229894 0.2115568 0.02458630 0.001463037 0.01933947 0.0007019570
## 109   109 0.03230183 0.2114575 0.02459496 0.001463231 0.01969204 0.0007036583
## 110   110 0.03230544 0.2113172 0.02459776 0.001461497 0.01957454 0.0006978891
## 111   111 0.03230995 0.2111259 0.02460000 0.001462234 0.01975481 0.0006997356
## 112   112 0.03231753 0.2108198 0.02460973 0.001457035 0.01989509 0.0006998023
## 113   113 0.03232586 0.2104785 0.02461977 0.001461560 0.02009914 0.0007025938
## 114   114 0.03233278 0.2101921 0.02462193 0.001461404 0.02024731 0.0007088097
## 115   115 0.03234308 0.2097399 0.02463340 0.001456815 0.02012839 0.0007032015
## 116   116 0.03234164 0.2098275 0.02463260 0.001452307 0.02008379 0.0006989580
## 117   117 0.03233844 0.2099830 0.02462906 0.001449086 0.02017131 0.0006965985
## 118   118 0.03233952 0.2099564 0.02462969 0.001445617 0.02029948 0.0006974606
## 119   119 0.03234053 0.2099091 0.02462938 0.001445084 0.02024108 0.0006953109
## 120   120 0.03234679 0.2096605 0.02463224 0.001446224 0.01968261 0.0006976714
## 121   121 0.03235391 0.2093846 0.02464147 0.001449870 0.01956157 0.0006978373
## 122   122 0.03235594 0.2092939 0.02464064 0.001456665 0.01959314 0.0006985586
## 123   123 0.03236312 0.2089893 0.02464125 0.001450493 0.01966320 0.0006994121
## 124   124 0.03237005 0.2087040 0.02464672 0.001456670 0.01977802 0.0007110982
## 125   125 0.03236611 0.2088384 0.02464741 0.001451857 0.01982704 0.0007082578
## 126   126 0.03236983 0.2086513 0.02464855 0.001450608 0.01973157 0.0007070967
## 127   127 0.03236702 0.2087728 0.02465163 0.001443048 0.01959941 0.0006994268
## 128   128 0.03236813 0.2087285 0.02465265 0.001445830 0.01965970 0.0006990764
## 129   129 0.03236655 0.2088366 0.02464801 0.001451042 0.01987339 0.0007056639
## 130   130 0.03237284 0.2085957 0.02465581 0.001453114 0.02001560 0.0007051699
## 131   131 0.03237054 0.2086809 0.02465524 0.001454014 0.01978694 0.0007055681
## 132   132 0.03237433 0.2085338 0.02465740 0.001460649 0.02004828 0.0007093348
## 133   133 0.03237625 0.2084771 0.02465997 0.001463811 0.02001250 0.0007142110
## 134   134 0.03238063 0.2082721 0.02466247 0.001467982 0.02002332 0.0007184374
## 135   135 0.03238203 0.2082081 0.02466592 0.001467785 0.02028677 0.0007159898
## 136   136 0.03238183 0.2082432 0.02466867 0.001469559 0.02045894 0.0007220370
## 137   137 0.03238290 0.2081932 0.02466644 0.001468473 0.02062916 0.0007225481
## 138   138 0.03238504 0.2081194 0.02466681 0.001467524 0.02024962 0.0007245681
## 139   139 0.03237739 0.2084694 0.02465537 0.001472496 0.02044149 0.0007322944
## 140   140 0.03237728 0.2084929 0.02465460 0.001472548 0.02055625 0.0007287566
## 141   141 0.03237777 0.2085047 0.02465841 0.001477572 0.02068249 0.0007269164
## 142   142 0.03237144 0.2087702 0.02465787 0.001478753 0.02059352 0.0007300793
## 143   143 0.03236709 0.2089662 0.02465236 0.001479036 0.02076908 0.0007332516
## 144   144 0.03236829 0.2089042 0.02465616 0.001476186 0.02045209 0.0007344411
## 145   145 0.03236682 0.2089744 0.02465522 0.001483258 0.02066627 0.0007389862
## 146   146 0.03236760 0.2089431 0.02465746 0.001494223 0.02059667 0.0007473667
## 147   147 0.03236930 0.2088773 0.02466088 0.001497953 0.02075258 0.0007495153
## 148   148 0.03237061 0.2088295 0.02466086 0.001495624 0.02076994 0.0007480942
## 149   149 0.03236529 0.2091095 0.02465625 0.001496206 0.02085265 0.0007500828
## 150   150 0.03236393 0.2091907 0.02465641 0.001495397 0.02102532 0.0007522352
## 151   151 0.03236979 0.2089387 0.02466017 0.001501370 0.02110538 0.0007524256
## 152   152 0.03236973 0.2089355 0.02465720 0.001500666 0.02132627 0.0007513503
## 153   153 0.03236718 0.2090340 0.02465324 0.001493825 0.02119334 0.0007464709
## 154   154 0.03236945 0.2089463 0.02465645 0.001497245 0.02115764 0.0007497064
## 155   155 0.03237013 0.2089025 0.02465778 0.001500470 0.02121948 0.0007529308
## 156   156 0.03236836 0.2089725 0.02465733 0.001504294 0.02116572 0.0007579519
## 157   157 0.03236516 0.2091041 0.02465353 0.001500450 0.02115671 0.0007589187
## 158   158 0.03236346 0.2091861 0.02465100 0.001504278 0.02102376 0.0007603892
## 159   159 0.03235701 0.2094459 0.02464440 0.001499829 0.02108592 0.0007596713
## 160   160 0.03236302 0.2091735 0.02465265 0.001501556 0.02115614 0.0007615501
## 161   161 0.03236269 0.2092054 0.02465540 0.001500531 0.02110199 0.0007614088
## 162   162 0.03236487 0.2091079 0.02465757 0.001500332 0.02097270 0.0007628420
## 163   163 0.03236540 0.2090802 0.02465889 0.001502076 0.02097277 0.0007652958
## 164   164 0.03236529 0.2090745 0.02465965 0.001498505 0.02091540 0.0007606235
## 165   165 0.03236436 0.2091035 0.02465766 0.001498240 0.02086128 0.0007611603
## 166   166 0.03236213 0.2092025 0.02465804 0.001498770 0.02077295 0.0007598125
## 167   167 0.03236567 0.2090544 0.02466143 0.001494534 0.02057277 0.0007580261
## 168   168 0.03236425 0.2091252 0.02466047 0.001496216 0.02053200 0.0007580967
## 169   169 0.03236673 0.2090204 0.02466214 0.001490475 0.02032829 0.0007511719
## 170   170 0.03236852 0.2089594 0.02466545 0.001490021 0.02043156 0.0007546072
## 171   171 0.03236449 0.2091472 0.02466368 0.001489977 0.02042619 0.0007522308
## 172   172 0.03236568 0.2090999 0.02466471 0.001492730 0.02042618 0.0007540343
## 173   173 0.03236609 0.2090973 0.02466759 0.001493358 0.02042529 0.0007555700
## 174   174 0.03237287 0.2088194 0.02467662 0.001495079 0.02042392 0.0007562069
## 175   175 0.03237492 0.2087234 0.02467629 0.001494200 0.02035672 0.0007546908
## 176   176 0.03237735 0.2086387 0.02467618 0.001495720 0.02030256 0.0007553836
## 177   177 0.03238092 0.2084925 0.02468078 0.001492784 0.02014192 0.0007516585
## 178   178 0.03238172 0.2084696 0.02468156 0.001490689 0.01996088 0.0007519556
## 179   179 0.03237793 0.2086221 0.02467669 0.001492583 0.01999650 0.0007508839
## 180   180 0.03237553 0.2087111 0.02467452 0.001494348 0.02008713 0.0007499132
## 181   181 0.03237855 0.2085856 0.02467722 0.001491880 0.01987468 0.0007466168
## 182   182 0.03237788 0.2086077 0.02467599 0.001491385 0.01981952 0.0007450645
## 183   183 0.03237954 0.2085384 0.02467603 0.001491164 0.01972100 0.0007418197
## 184   184 0.03238097 0.2084817 0.02467801 0.001488487 0.01958478 0.0007410066
## 185   185 0.03238309 0.2083890 0.02467835 0.001486525 0.01955053 0.0007405120
## 186   186 0.03238640 0.2082532 0.02467898 0.001484660 0.01956794 0.0007396025
## 187   187 0.03238687 0.2082370 0.02467856 0.001483353 0.01946381 0.0007377509
## 188   188 0.03238918 0.2081426 0.02467944 0.001483852 0.01946909 0.0007387150
## 189   189 0.03239092 0.2080661 0.02468004 0.001482983 0.01929234 0.0007384544
## 190   190 0.03239467 0.2079081 0.02468424 0.001481706 0.01924843 0.0007378820
## 191   191 0.03239618 0.2078525 0.02468530 0.001480271 0.01921047 0.0007373179
## 192   192 0.03239630 0.2078613 0.02468415 0.001479923 0.01928535 0.0007384786
## 193   193 0.03239697 0.2078304 0.02468454 0.001479089 0.01922801 0.0007386861
## 194   194 0.03239506 0.2079116 0.02468327 0.001480461 0.01913816 0.0007380782
## 195   195 0.03239522 0.2079060 0.02468292 0.001479957 0.01911824 0.0007385234
## 196   196 0.03239598 0.2078742 0.02468396 0.001480230 0.01912957 0.0007398277
## 197   197 0.03239837 0.2077702 0.02468658 0.001479692 0.01911349 0.0007392401
## 198   198 0.03239672 0.2078366 0.02468530 0.001480818 0.01911019 0.0007403815
## 199   199 0.03239843 0.2077671 0.02468512 0.001480140 0.01907493 0.0007409087
## 200   200 0.03239709 0.2078246 0.02468348 0.001481060 0.01914102 0.0007399355
## 201   201 0.03239598 0.2078783 0.02468274 0.001482422 0.01914546 0.0007402610
## 202   202 0.03239411 0.2079638 0.02468318 0.001482194 0.01910760 0.0007398654
## 203   203 0.03239621 0.2078847 0.02468393 0.001481791 0.01901557 0.0007403495
## 204   204 0.03239782 0.2078232 0.02468303 0.001482263 0.01911065 0.0007409477
## 205   205 0.03239813 0.2078171 0.02468274 0.001481333 0.01903385 0.0007396643
## 206   206 0.03239916 0.2077721 0.02468327 0.001484107 0.01913457 0.0007416513
## 207   207 0.03239912 0.2077756 0.02468328 0.001483192 0.01916391 0.0007408675
## 208   208 0.03239768 0.2078401 0.02468274 0.001483688 0.01922151 0.0007412517
## 209   209 0.03239804 0.2078230 0.02468236 0.001482209 0.01916628 0.0007398635
## 210   210 0.03239803 0.2078255 0.02468221 0.001481846 0.01917665 0.0007388410
## 211   211 0.03239735 0.2078514 0.02468182 0.001481697 0.01915682 0.0007380946
## 212   212 0.03239737 0.2078444 0.02468275 0.001480836 0.01913159 0.0007377400
## 213   213 0.03239689 0.2078739 0.02468365 0.001480841 0.01913289 0.0007379151
## 214   214 0.03239783 0.2078345 0.02468510 0.001481113 0.01914131 0.0007377816
## 215   215 0.03239708 0.2078651 0.02468436 0.001480911 0.01910152 0.0007371423
## 216   216 0.03239731 0.2078526 0.02468586 0.001480769 0.01909326 0.0007370225
## 217   217 0.03239835 0.2078074 0.02468653 0.001480460 0.01910234 0.0007368115
## 218   218 0.03239850 0.2078052 0.02468599 0.001480434 0.01908601 0.0007366686
## 219   219 0.03239870 0.2077968 0.02468589 0.001481235 0.01909185 0.0007371097
## 220   220 0.03239777 0.2078409 0.02468520 0.001481397 0.01909433 0.0007383870
## 221   221 0.03239771 0.2078455 0.02468459 0.001480940 0.01911086 0.0007386547
## 222   222 0.03239641 0.2079001 0.02468384 0.001480028 0.01910106 0.0007384054
## 223   223 0.03239602 0.2079170 0.02468327 0.001480344 0.01913320 0.0007386912
## 224   224 0.03239620 0.2079104 0.02468292 0.001480669 0.01912358 0.0007390457
## 225   225 0.03239558 0.2079363 0.02468282 0.001481830 0.01916216 0.0007403572
## 226   226 0.03239548 0.2079394 0.02468300 0.001481755 0.01918802 0.0007406822
## 227   227 0.03239493 0.2079645 0.02468266 0.001481883 0.01918029 0.0007405759
## 228   228 0.03239388 0.2080087 0.02468201 0.001481176 0.01915911 0.0007403683
## 229   229 0.03239372 0.2080174 0.02468183 0.001480870 0.01915881 0.0007401023
## 230   230 0.03239384 0.2080118 0.02468183 0.001480985 0.01915371 0.0007400824
## 231   231 0.03239376 0.2080148 0.02468183 0.001481094 0.01913600 0.0007400396
## 232   232 0.03239397 0.2080042 0.02468193 0.001481021 0.01912858 0.0007398989
## 233   233 0.03239415 0.2079983 0.02468221 0.001481044 0.01912868 0.0007399265
## 234   234 0.03239440 0.2079879 0.02468262 0.001481325 0.01914004 0.0007401556
## 235   235 0.03239422 0.2079947 0.02468262 0.001481258 0.01913714 0.0007400686
## 236   236 0.03239430 0.2079919 0.02468277 0.001481382 0.01914093 0.0007400643
## 237   237 0.03239448 0.2079846 0.02468290 0.001481298 0.01914150 0.0007400602
## 238   238 0.03239441 0.2079881 0.02468286 0.001481402 0.01914274 0.0007401495
## 239   239 0.03239443 0.2079871 0.02468302 0.001481352 0.01914633 0.0007400931
## 240   240 0.03239445 0.2079861 0.02468299 0.001481330 0.01914509 0.0007400672
## [1] "Best Model"
##    nvmax
## 19    19

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  1.988237e+00  1.980581e+00  1.995894e+00
## x4          -4.321837e-05 -6.068012e-05 -2.575661e-05
## x7           1.118167e-02  9.953279e-03  1.241005e-02
## x8           4.604115e-04  1.728983e-04  7.479248e-04
## x9           3.144328e-03  2.500333e-03  3.788324e-03
## x10          1.051023e-03  4.549996e-04  1.647046e-03
## x16          1.019789e-03  6.076877e-04  1.431891e-03
## x17          1.632689e-03  1.001442e-03  2.263935e-03
## x21          1.432126e-04  6.151940e-05  2.249057e-04
## stat13      -7.132321e-04 -1.189964e-03 -2.364999e-04
## stat14      -8.569270e-04 -1.332110e-03 -3.817434e-04
## stat23       6.971062e-04  2.190595e-04  1.175153e-03
## stat24      -7.739189e-04 -1.254354e-03 -2.934836e-04
## stat60       5.992247e-04  1.187520e-04  1.079697e-03
## stat91      -6.246528e-04 -1.101914e-03 -1.473917e-04
## stat98       3.559182e-03  3.087007e-03  4.031356e-03
## stat110     -3.508962e-03 -3.988726e-03 -3.029198e-03
## stat149     -7.085899e-04 -1.191292e-03 -2.258873e-04
## stat195      6.602235e-04  1.830305e-04  1.137416e-03
## x18.sqrt     2.617529e-02  2.433567e-02  2.801490e-02

Test

if (algo.forward.caret == TRUE){
    test.model(model=model.forward, test=data.test
             ,method = 'leapForward',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.038   2.084   2.097   2.097   2.110   2.147 
## [1] "leapForward  Test MSE: 0.000964626165422911"

Backward Elimination with CV

Train

if (algo.backward.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "leapBackward"
                                   ,feature.names =  feature.names)
  model.backward = returned$model
  id = returned$id
}
## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 19 on full training set
## [1] "All models results"
##     nvmax       RMSE  Rsquared        MAE      RMSESD RsquaredSD        MAESD
## 1       1 0.03429406 0.1067604 0.02659566 0.001380514 0.01920345 0.0007111804
## 2       2 0.03345922 0.1507056 0.02585671 0.001513502 0.02228332 0.0007636527
## 3       3 0.03305704 0.1708944 0.02540433 0.001593231 0.02013441 0.0008673065
## 4       4 0.03229670 0.2080695 0.02452928 0.001547612 0.01984658 0.0008543763
## 5       5 0.03204362 0.2202654 0.02436176 0.001548044 0.01918191 0.0008650394
## 6       6 0.03204567 0.2200958 0.02436311 0.001523974 0.01873001 0.0008244408
## 7       7 0.03198700 0.2229757 0.02434475 0.001496667 0.01767578 0.0007959974
## 8       8 0.03185614 0.2293116 0.02423250 0.001488678 0.01736199 0.0007882918
## 9       9 0.03189937 0.2272611 0.02425828 0.001483579 0.01744927 0.0007905599
## 10     10 0.03191321 0.2265856 0.02425885 0.001462645 0.01672747 0.0007696399
## 11     11 0.03190249 0.2271851 0.02427821 0.001470544 0.01706492 0.0007674826
## 12     12 0.03188801 0.2279486 0.02427609 0.001476996 0.01703738 0.0007809244
## 13     13 0.03187006 0.2288751 0.02426647 0.001475758 0.01645541 0.0007624986
## 14     14 0.03181575 0.2314520 0.02424750 0.001462587 0.01637929 0.0007558458
## 15     15 0.03182362 0.2311417 0.02424614 0.001477059 0.01817657 0.0007733136
## 16     16 0.03182072 0.2312796 0.02423546 0.001485069 0.01891176 0.0007738498
## 17     17 0.03178287 0.2330225 0.02419076 0.001467648 0.01823470 0.0007503516
## 18     18 0.03176840 0.2336788 0.02417930 0.001469475 0.01864076 0.0007492256
## 19     19 0.03176594 0.2337762 0.02417991 0.001454459 0.01730803 0.0007501501
## 20     20 0.03178131 0.2330895 0.02419517 0.001452649 0.01689762 0.0007591666
## 21     21 0.03180199 0.2321759 0.02420844 0.001447631 0.01702310 0.0007569351
## 22     22 0.03180617 0.2320464 0.02421929 0.001463020 0.01786903 0.0007578223
## 23     23 0.03183230 0.2309227 0.02423681 0.001475969 0.01840329 0.0007719206
## 24     24 0.03184653 0.2302875 0.02424625 0.001473999 0.01833435 0.0007599518
## 25     25 0.03184783 0.2302468 0.02423764 0.001478678 0.01929429 0.0007770928
## 26     26 0.03186320 0.2295546 0.02424442 0.001485252 0.01889201 0.0007767636
## 27     27 0.03186264 0.2296512 0.02423935 0.001490441 0.01915020 0.0007689690
## 28     28 0.03186146 0.2297167 0.02422987 0.001475306 0.01882703 0.0007493786
## 29     29 0.03186481 0.2295912 0.02424238 0.001497889 0.01993094 0.0007636074
## 30     30 0.03187147 0.2293405 0.02424066 0.001502196 0.01918814 0.0007684686
## 31     31 0.03186795 0.2295203 0.02423052 0.001507154 0.01891236 0.0007743578
## 32     32 0.03187245 0.2292979 0.02423140 0.001521077 0.01861607 0.0007840075
## 33     33 0.03189498 0.2282986 0.02424300 0.001533510 0.01902505 0.0007959202
## 34     34 0.03191475 0.2273992 0.02425717 0.001527371 0.01866901 0.0007910417
## 35     35 0.03191973 0.2272136 0.02426747 0.001548638 0.01915964 0.0007976372
## 36     36 0.03191995 0.2272271 0.02427744 0.001554435 0.01943801 0.0007985738
## 37     37 0.03194419 0.2261589 0.02429952 0.001553932 0.01989543 0.0008060600
## 38     38 0.03194892 0.2259436 0.02430600 0.001541979 0.02030290 0.0008000740
## 39     39 0.03196447 0.2252532 0.02431617 0.001543123 0.02058447 0.0008094333
## 40     40 0.03196572 0.2252230 0.02431632 0.001552867 0.02098727 0.0008098937
## 41     41 0.03197143 0.2249977 0.02432398 0.001569534 0.02205309 0.0008227816
## 42     42 0.03197208 0.2250018 0.02432636 0.001568253 0.02210264 0.0008308312
## 43     43 0.03197259 0.2250081 0.02433498 0.001558467 0.02206258 0.0008263546
## 44     44 0.03198409 0.2245070 0.02434425 0.001553235 0.02207327 0.0008203153
## 45     45 0.03199186 0.2242004 0.02435021 0.001552059 0.02258466 0.0008244461
## 46     46 0.03198156 0.2246913 0.02434170 0.001541737 0.02254903 0.0008218987
## 47     47 0.03198436 0.2245712 0.02434372 0.001544313 0.02227288 0.0008245223
## 48     48 0.03198887 0.2243473 0.02434410 0.001530626 0.02154797 0.0008072368
## 49     49 0.03200445 0.2236660 0.02434874 0.001516282 0.02041052 0.0007824247
## 50     50 0.03200881 0.2234880 0.02435206 0.001528848 0.02035281 0.0007958444
## 51     51 0.03202549 0.2227279 0.02437221 0.001515513 0.01923045 0.0007877430
## 52     52 0.03203202 0.2224621 0.02436672 0.001511087 0.01913791 0.0007725501
## 53     53 0.03203330 0.2224706 0.02436533 0.001505063 0.01877313 0.0007712752
## 54     54 0.03203215 0.2225339 0.02435727 0.001501146 0.01848334 0.0007662415
## 55     55 0.03204350 0.2220340 0.02436873 0.001511998 0.01827103 0.0007684908
## 56     56 0.03205791 0.2214585 0.02437369 0.001516104 0.01796123 0.0007715864
## 57     57 0.03205359 0.2216561 0.02437348 0.001522466 0.01847252 0.0007702584
## 58     58 0.03206838 0.2210071 0.02438300 0.001533167 0.01892576 0.0007797491
## 59     59 0.03207834 0.2205912 0.02439289 0.001548350 0.01983204 0.0007849236
## 60     60 0.03208301 0.2203700 0.02440020 0.001541650 0.01958320 0.0007793204
## 61     61 0.03209447 0.2198412 0.02440870 0.001536107 0.01923108 0.0007787927
## 62     62 0.03210484 0.2193747 0.02442038 0.001531409 0.01940974 0.0007665053
## 63     63 0.03210223 0.2195230 0.02442609 0.001528067 0.01955342 0.0007653653
## 64     64 0.03210266 0.2195207 0.02442814 0.001526779 0.01956283 0.0007656588
## 65     65 0.03211893 0.2188535 0.02443801 0.001516847 0.01923860 0.0007478403
## 66     66 0.03212122 0.2187686 0.02443325 0.001506718 0.01876673 0.0007342807
## 67     67 0.03213486 0.2181743 0.02444381 0.001500265 0.01857931 0.0007343226
## 68     68 0.03213678 0.2181300 0.02444142 0.001504928 0.01859215 0.0007381114
## 69     69 0.03214033 0.2179808 0.02443565 0.001501438 0.01882626 0.0007365117
## 70     70 0.03214902 0.2176500 0.02444908 0.001488004 0.01868270 0.0007235710
## 71     71 0.03215468 0.2174029 0.02444930 0.001495121 0.01862773 0.0007295412
## 72     72 0.03216105 0.2171403 0.02446326 0.001503901 0.01887213 0.0007414244
## 73     73 0.03216185 0.2170945 0.02446157 0.001499686 0.01845792 0.0007452694
## 74     74 0.03217534 0.2164831 0.02447950 0.001496848 0.01859621 0.0007416342
## 75     75 0.03217701 0.2163884 0.02449519 0.001485773 0.01835885 0.0007348918
## 76     76 0.03216468 0.2168823 0.02449000 0.001479867 0.01804348 0.0007313064
## 77     77 0.03217884 0.2162752 0.02450319 0.001474032 0.01816058 0.0007261092
## 78     78 0.03218384 0.2160636 0.02450436 0.001480327 0.01817902 0.0007239309
## 79     79 0.03218702 0.2159115 0.02450538 0.001488547 0.01843849 0.0007363910
## 80     80 0.03219392 0.2156280 0.02451474 0.001491183 0.01878467 0.0007319049
## 81     81 0.03220500 0.2151800 0.02451873 0.001499556 0.01906022 0.0007356225
## 82     82 0.03221885 0.2146028 0.02452966 0.001498137 0.01875068 0.0007320122
## 83     83 0.03222286 0.2144290 0.02452378 0.001487101 0.01847021 0.0007224231
## 84     84 0.03222887 0.2141875 0.02452599 0.001489800 0.01822029 0.0007302808
## 85     85 0.03222729 0.2142440 0.02452642 0.001482265 0.01836321 0.0007265580
## 86     86 0.03223520 0.2139317 0.02452962 0.001473228 0.01842020 0.0007193060
## 87     87 0.03224161 0.2136547 0.02453779 0.001463040 0.01836216 0.0007088130
## 88     88 0.03224449 0.2135323 0.02454082 0.001456241 0.01839350 0.0007006543
## 89     89 0.03225416 0.2131328 0.02454678 0.001461613 0.01824106 0.0007067223
## 90     90 0.03226375 0.2127671 0.02455036 0.001461534 0.01844994 0.0007105263
## 91     91 0.03225760 0.2130676 0.02454387 0.001471421 0.01880264 0.0007153715
## 92     92 0.03224683 0.2135719 0.02453502 0.001479065 0.01922443 0.0007140951
## 93     93 0.03224697 0.2135886 0.02453403 0.001480040 0.01942316 0.0007180390
## 94     94 0.03224190 0.2138262 0.02452517 0.001480372 0.01941036 0.0007159767
## 95     95 0.03225370 0.2132918 0.02452807 0.001474050 0.01917278 0.0007044508
## 96     96 0.03225614 0.2132146 0.02453348 0.001472934 0.01933172 0.0007055483
## 97     97 0.03225578 0.2132057 0.02453516 0.001472622 0.01944929 0.0007027396
## 98     98 0.03225797 0.2131154 0.02454040 0.001467217 0.01901681 0.0006918668
## 99     99 0.03225732 0.2131051 0.02453938 0.001464273 0.01895694 0.0006956634
## 100   100 0.03226897 0.2126059 0.02455250 0.001468578 0.01936556 0.0006980707
## 101   101 0.03227075 0.2125875 0.02455425 0.001467009 0.01954170 0.0006954164
## 102   102 0.03227465 0.2124562 0.02455557 0.001469923 0.01998641 0.0007012370
## 103   103 0.03226768 0.2127802 0.02455526 0.001473163 0.02003555 0.0007143576
## 104   104 0.03226669 0.2128588 0.02455281 0.001468232 0.02010288 0.0007152007
## 105   105 0.03227134 0.2126997 0.02455953 0.001472400 0.02012718 0.0007195177
## 106   106 0.03228004 0.2123220 0.02457038 0.001480891 0.02004164 0.0007182579
## 107   107 0.03229131 0.2118317 0.02458335 0.001484355 0.02006029 0.0007209599
## 108   108 0.03230523 0.2112235 0.02459674 0.001473535 0.02007713 0.0007132814
## 109   109 0.03230431 0.2113113 0.02459842 0.001471905 0.02006857 0.0007134567
## 110   110 0.03231223 0.2109911 0.02460145 0.001454071 0.02001363 0.0006984559
## 111   111 0.03231652 0.2108279 0.02460678 0.001457384 0.02012092 0.0007020681
## 112   112 0.03232142 0.2106073 0.02461091 0.001459528 0.02021721 0.0007117102
## 113   113 0.03232274 0.2105522 0.02461594 0.001458815 0.02056517 0.0007105386
## 114   114 0.03233046 0.2102671 0.02462250 0.001453761 0.02058252 0.0007089183
## 115   115 0.03233732 0.2099832 0.02462877 0.001456266 0.02033263 0.0007106970
## 116   116 0.03233432 0.2101634 0.02462295 0.001455521 0.02058871 0.0007058468
## 117   117 0.03233828 0.2100358 0.02462476 0.001454532 0.02077079 0.0007056218
## 118   118 0.03234528 0.2097407 0.02462673 0.001453299 0.02037204 0.0006995243
## 119   119 0.03235161 0.2094912 0.02463358 0.001452309 0.02032882 0.0007021150
## 120   120 0.03235682 0.2092746 0.02464070 0.001455047 0.02008053 0.0007104449
## 121   121 0.03235608 0.2092965 0.02463963 0.001455997 0.01997193 0.0007073868
## 122   122 0.03235113 0.2095351 0.02463956 0.001459374 0.02014841 0.0007105557
## 123   123 0.03236407 0.2089745 0.02464900 0.001455128 0.02002467 0.0007106562
## 124   124 0.03236133 0.2090882 0.02464469 0.001452741 0.01990324 0.0007117433
## 125   125 0.03236114 0.2090539 0.02464388 0.001450758 0.01996345 0.0007108330
## 126   126 0.03236275 0.2089530 0.02464425 0.001444367 0.01978153 0.0007016554
## 127   127 0.03236284 0.2089352 0.02465054 0.001445330 0.01985335 0.0006994935
## 128   128 0.03236343 0.2089270 0.02464948 0.001445319 0.01984544 0.0006974992
## 129   129 0.03236625 0.2088284 0.02464956 0.001446685 0.02001411 0.0006966764
## 130   130 0.03236740 0.2087767 0.02464985 0.001448439 0.02007970 0.0006994050
## 131   131 0.03237094 0.2086349 0.02465800 0.001449872 0.02005047 0.0007033492
## 132   132 0.03237700 0.2083854 0.02466112 0.001457353 0.01995447 0.0007072227
## 133   133 0.03237397 0.2085217 0.02465947 0.001461852 0.02018272 0.0007127523
## 134   134 0.03237833 0.2083333 0.02466164 0.001466098 0.02017548 0.0007182871
## 135   135 0.03237486 0.2084846 0.02465706 0.001468636 0.02028004 0.0007203183
## 136   136 0.03237603 0.2084509 0.02465940 0.001469024 0.02035791 0.0007239880
## 137   137 0.03237855 0.2083729 0.02466099 0.001464965 0.02049925 0.0007220875
## 138   138 0.03238134 0.2082858 0.02466407 0.001473336 0.02059620 0.0007346973
## 139   139 0.03237641 0.2085015 0.02465568 0.001470559 0.02054103 0.0007315036
## 140   140 0.03237327 0.2086603 0.02465339 0.001473233 0.02046302 0.0007272745
## 141   141 0.03237200 0.2087621 0.02465581 0.001478080 0.02068077 0.0007289863
## 142   142 0.03236508 0.2090498 0.02465206 0.001478112 0.02065351 0.0007322853
## 143   143 0.03236268 0.2091681 0.02464973 0.001478813 0.02089479 0.0007357185
## 144   144 0.03237044 0.2088270 0.02465806 0.001479763 0.02060053 0.0007368362
## 145   145 0.03236723 0.2089481 0.02465546 0.001483459 0.02063792 0.0007395776
## 146   146 0.03236697 0.2089552 0.02465965 0.001493556 0.02064058 0.0007457151
## 147   147 0.03236946 0.2088568 0.02466074 0.001498910 0.02083255 0.0007510473
## 148   148 0.03237397 0.2086797 0.02466409 0.001496492 0.02062100 0.0007474982
## 149   149 0.03236917 0.2089277 0.02466011 0.001496790 0.02077144 0.0007492073
## 150   150 0.03237020 0.2089014 0.02466027 0.001495460 0.02095101 0.0007507552
## 151   151 0.03236952 0.2089414 0.02465838 0.001500593 0.02110873 0.0007502705
## 152   152 0.03236386 0.2091791 0.02465153 0.001500150 0.02117570 0.0007493063
## 153   153 0.03236442 0.2091484 0.02465244 0.001494917 0.02100680 0.0007470865
## 154   154 0.03236756 0.2090013 0.02465672 0.001496593 0.02102726 0.0007473661
## 155   155 0.03237111 0.2088441 0.02465753 0.001501023 0.02109288 0.0007510756
## 156   156 0.03236760 0.2089919 0.02465805 0.001504526 0.02110436 0.0007579954
## 157   157 0.03236667 0.2090349 0.02465568 0.001501764 0.02096040 0.0007601112
## 158   158 0.03236129 0.2092769 0.02465130 0.001503064 0.02101718 0.0007599487
## 159   159 0.03235830 0.2093909 0.02464771 0.001500399 0.02097560 0.0007610789
## 160   160 0.03236374 0.2091386 0.02465252 0.001502711 0.02102676 0.0007625560
## 161   161 0.03236300 0.2091772 0.02465391 0.001501013 0.02110506 0.0007614320
## 162   162 0.03236304 0.2091723 0.02465781 0.001498680 0.02094622 0.0007600571
## 163   163 0.03236190 0.2092251 0.02465845 0.001498103 0.02086113 0.0007606414
## 164   164 0.03236433 0.2091095 0.02465962 0.001496423 0.02079047 0.0007564593
## 165   165 0.03236367 0.2091221 0.02465900 0.001496091 0.02075334 0.0007567676
## 166   166 0.03236293 0.2091482 0.02465881 0.001490900 0.02056858 0.0007540689
## 167   167 0.03236525 0.2090558 0.02466263 0.001492389 0.02050543 0.0007564942
## 168   168 0.03236193 0.2092171 0.02465983 0.001496006 0.02058812 0.0007582883
## 169   169 0.03236699 0.2090077 0.02466343 0.001490528 0.02031474 0.0007504400
## 170   170 0.03236808 0.2089662 0.02466631 0.001490986 0.02024752 0.0007539098
## 171   171 0.03236853 0.2089601 0.02466713 0.001491621 0.02023319 0.0007519871
## 172   172 0.03236822 0.2089844 0.02466754 0.001495242 0.02038957 0.0007556653
## 173   173 0.03237254 0.2088151 0.02467288 0.001498946 0.02036924 0.0007602718
## 174   174 0.03237394 0.2087724 0.02467558 0.001498152 0.02051220 0.0007579239
## 175   175 0.03237621 0.2086811 0.02467742 0.001495475 0.02043306 0.0007561694
## 176   176 0.03237824 0.2086114 0.02467864 0.001494288 0.02022036 0.0007543052
## 177   177 0.03237720 0.2086512 0.02467622 0.001492277 0.02020864 0.0007535000
## 178   178 0.03237802 0.2086301 0.02467643 0.001490715 0.02013652 0.0007531552
## 179   179 0.03237679 0.2086704 0.02467515 0.001493202 0.02018990 0.0007512603
## 180   180 0.03237416 0.2087671 0.02467474 0.001492923 0.02006189 0.0007510425
## 181   181 0.03237839 0.2085954 0.02467616 0.001492758 0.01995077 0.0007472509
## 182   182 0.03237773 0.2086193 0.02467459 0.001492440 0.01991178 0.0007462513
## 183   183 0.03237796 0.2086080 0.02467450 0.001490803 0.01979276 0.0007426946
## 184   184 0.03238097 0.2084817 0.02467801 0.001488487 0.01958478 0.0007410066
## 185   185 0.03238309 0.2083890 0.02467835 0.001486525 0.01955053 0.0007405120
## 186   186 0.03238640 0.2082532 0.02467898 0.001484660 0.01956794 0.0007396025
## 187   187 0.03238743 0.2082126 0.02467934 0.001483784 0.01946153 0.0007384450
## 188   188 0.03238977 0.2081168 0.02468028 0.001484310 0.01946686 0.0007394676
## 189   189 0.03239206 0.2080207 0.02468198 0.001484350 0.01932416 0.0007405500
## 190   190 0.03239534 0.2078856 0.02468487 0.001482727 0.01928358 0.0007387889
## 191   191 0.03239618 0.2078525 0.02468530 0.001480271 0.01921047 0.0007373179
## 192   192 0.03239654 0.2078454 0.02468455 0.001478649 0.01927328 0.0007366974
## 193   193 0.03239613 0.2078628 0.02468274 0.001478695 0.01919531 0.0007367794
## 194   194 0.03239506 0.2079116 0.02468327 0.001480461 0.01913816 0.0007380782
## 195   195 0.03239502 0.2079147 0.02468302 0.001480502 0.01913387 0.0007382654
## 196   196 0.03239750 0.2078113 0.02468465 0.001480713 0.01912587 0.0007401665
## 197   197 0.03239924 0.2077398 0.02468675 0.001479507 0.01913910 0.0007394008
## 198   198 0.03239864 0.2077563 0.02468584 0.001481117 0.01905466 0.0007404648
## 199   199 0.03239735 0.2078081 0.02468488 0.001480405 0.01901357 0.0007408789
## 200   200 0.03239605 0.2078678 0.02468340 0.001481194 0.01910648 0.0007399108
## 201   201 0.03239631 0.2078673 0.02468358 0.001482518 0.01913376 0.0007407182
## 202   202 0.03239485 0.2079343 0.02468363 0.001482411 0.01907671 0.0007401114
## 203   203 0.03239621 0.2078847 0.02468393 0.001481791 0.01901557 0.0007403495
## 204   204 0.03239784 0.2078263 0.02468301 0.001482253 0.01910699 0.0007409573
## 205   205 0.03239817 0.2078197 0.02468277 0.001481319 0.01903073 0.0007396540
## 206   206 0.03239920 0.2077747 0.02468324 0.001484092 0.01913154 0.0007416637
## 207   207 0.03239912 0.2077756 0.02468328 0.001483192 0.01916391 0.0007408675
## 208   208 0.03239768 0.2078401 0.02468274 0.001483688 0.01922151 0.0007412517
## 209   209 0.03239804 0.2078230 0.02468236 0.001482209 0.01916628 0.0007398635
## 210   210 0.03239803 0.2078255 0.02468221 0.001481846 0.01917665 0.0007388410
## 211   211 0.03239735 0.2078514 0.02468182 0.001481697 0.01915682 0.0007380946
## 212   212 0.03239686 0.2078656 0.02468206 0.001480059 0.01909814 0.0007367458
## 213   213 0.03239639 0.2078954 0.02468304 0.001480077 0.01909881 0.0007370320
## 214   214 0.03239698 0.2078692 0.02468424 0.001479809 0.01908598 0.0007365311
## 215   215 0.03239708 0.2078651 0.02468436 0.001480911 0.01910152 0.0007371423
## 216   216 0.03239740 0.2078470 0.02468581 0.001480816 0.01909102 0.0007369855
## 217   217 0.03239844 0.2078016 0.02468647 0.001480508 0.01910005 0.0007367713
## 218   218 0.03239850 0.2078052 0.02468599 0.001480434 0.01908601 0.0007366686
## 219   219 0.03239870 0.2077968 0.02468589 0.001481235 0.01909185 0.0007371097
## 220   220 0.03239777 0.2078409 0.02468520 0.001481397 0.01909433 0.0007383870
## 221   221 0.03239771 0.2078455 0.02468459 0.001480940 0.01911086 0.0007386547
## 222   222 0.03239636 0.2079011 0.02468383 0.001480000 0.01910149 0.0007384031
## 223   223 0.03239645 0.2078970 0.02468368 0.001480565 0.01912517 0.0007389737
## 224   224 0.03239620 0.2079104 0.02468292 0.001480669 0.01912358 0.0007390457
## 225   225 0.03239558 0.2079363 0.02468282 0.001481830 0.01916216 0.0007403572
## 226   226 0.03239548 0.2079394 0.02468300 0.001481755 0.01918802 0.0007406822
## 227   227 0.03239493 0.2079645 0.02468266 0.001481883 0.01918029 0.0007405759
## 228   228 0.03239388 0.2080087 0.02468201 0.001481176 0.01915911 0.0007403683
## 229   229 0.03239372 0.2080174 0.02468183 0.001480870 0.01915881 0.0007401023
## 230   230 0.03239384 0.2080118 0.02468183 0.001480985 0.01915371 0.0007400824
## 231   231 0.03239376 0.2080148 0.02468183 0.001481094 0.01913600 0.0007400396
## 232   232 0.03239397 0.2080042 0.02468193 0.001481021 0.01912858 0.0007398989
## 233   233 0.03239415 0.2079983 0.02468221 0.001481044 0.01912868 0.0007399265
## 234   234 0.03239440 0.2079879 0.02468262 0.001481325 0.01914004 0.0007401556
## 235   235 0.03239422 0.2079947 0.02468262 0.001481258 0.01913714 0.0007400686
## 236   236 0.03239430 0.2079919 0.02468277 0.001481382 0.01914093 0.0007400643
## 237   237 0.03239448 0.2079846 0.02468290 0.001481298 0.01914150 0.0007400602
## 238   238 0.03239441 0.2079881 0.02468286 0.001481402 0.01914274 0.0007401495
## 239   239 0.03239443 0.2079871 0.02468302 0.001481352 0.01914633 0.0007400931
## 240   240 0.03239445 0.2079861 0.02468299 0.001481330 0.01914509 0.0007400672
## [1] "Best Model"
##    nvmax
## 19    19

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  1.988237e+00  1.980581e+00  1.995894e+00
## x4          -4.321837e-05 -6.068012e-05 -2.575661e-05
## x7           1.118167e-02  9.953279e-03  1.241005e-02
## x8           4.604115e-04  1.728983e-04  7.479248e-04
## x9           3.144328e-03  2.500333e-03  3.788324e-03
## x10          1.051023e-03  4.549996e-04  1.647046e-03
## x16          1.019789e-03  6.076877e-04  1.431891e-03
## x17          1.632689e-03  1.001442e-03  2.263935e-03
## x21          1.432126e-04  6.151940e-05  2.249057e-04
## stat13      -7.132321e-04 -1.189964e-03 -2.364999e-04
## stat14      -8.569270e-04 -1.332110e-03 -3.817434e-04
## stat23       6.971062e-04  2.190595e-04  1.175153e-03
## stat24      -7.739189e-04 -1.254354e-03 -2.934836e-04
## stat60       5.992247e-04  1.187520e-04  1.079697e-03
## stat91      -6.246528e-04 -1.101914e-03 -1.473917e-04
## stat98       3.559182e-03  3.087007e-03  4.031356e-03
## stat110     -3.508962e-03 -3.988726e-03 -3.029198e-03
## stat149     -7.085899e-04 -1.191292e-03 -2.258873e-04
## stat195      6.602235e-04  1.830305e-04  1.137416e-03
## x18.sqrt     2.617529e-02  2.433567e-02  2.801490e-02

Test

if (algo.backward.caret == TRUE){
  test.model(model.backward, data.test
             ,method = 'leapBackward',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.038   2.084   2.097   2.097   2.110   2.147 
## [1] "leapBackward  Test MSE: 0.000964626165422911"

Stepwise Selection with CV

Train

if (algo.stepwise.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "leapSeq"
                                   ,feature.names = feature.names)
  model.stepwise = returned$model
  id = returned$id
}
## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 19 on full training set
## [1] "All models results"
##     nvmax       RMSE  Rsquared        MAE       RMSESD RsquaredSD        MAESD
## 1       1 0.03429406 0.1067604 0.02659566 0.0013805138 0.01920345 0.0007111804
## 2       2 0.03345922 0.1507056 0.02585671 0.0015135022 0.02228332 0.0007636527
## 3       3 0.03305704 0.1708944 0.02540433 0.0015932308 0.02013441 0.0008673065
## 4       4 0.03229670 0.2080695 0.02452928 0.0015476117 0.01984658 0.0008543763
## 5       5 0.03204362 0.2202654 0.02436176 0.0015480436 0.01918191 0.0008650394
## 6       6 0.03204567 0.2200958 0.02436311 0.0015239736 0.01873001 0.0008244408
## 7       7 0.03198700 0.2229757 0.02434475 0.0014966672 0.01767578 0.0007959974
## 8       8 0.03185614 0.2293116 0.02423250 0.0014886775 0.01736199 0.0007882918
## 9       9 0.03189937 0.2272611 0.02425828 0.0014835785 0.01744927 0.0007905599
## 10     10 0.03191321 0.2265856 0.02425885 0.0014626447 0.01672747 0.0007696399
## 11     11 0.03190249 0.2271851 0.02427821 0.0014705438 0.01706492 0.0007674826
## 12     12 0.03188801 0.2279486 0.02427609 0.0014769965 0.01703738 0.0007809244
## 13     13 0.03187006 0.2288751 0.02426647 0.0014757575 0.01645541 0.0007624986
## 14     14 0.03181575 0.2314520 0.02424750 0.0014625866 0.01637929 0.0007558458
## 15     15 0.03182362 0.2311417 0.02424614 0.0014770593 0.01817657 0.0007733136
## 16     16 0.03182072 0.2312796 0.02423546 0.0014850694 0.01891176 0.0007738498
## 17     17 0.03178287 0.2330225 0.02419076 0.0014676479 0.01823470 0.0007503516
## 18     18 0.03177599 0.2333205 0.02418540 0.0014745075 0.01864798 0.0007557786
## 19     19 0.03177516 0.2333527 0.02419055 0.0014589319 0.01813944 0.0007570440
## 20     20 0.03212935 0.2158744 0.02447857 0.0020008434 0.05718967 0.0013528862
## 21     21 0.03210167 0.2169292 0.02443221 0.0018166532 0.05815109 0.0011011837
## 22     22 0.03209848 0.2171453 0.02443501 0.0018282752 0.05835720 0.0010994512
## 23     23 0.03183230 0.2309227 0.02423681 0.0014759686 0.01840329 0.0007719206
## 24     24 0.03214197 0.2157085 0.02447254 0.0020893385 0.05740007 0.0012717717
## 25     25 0.03184699 0.2302590 0.02423858 0.0014760023 0.01809973 0.0007705003
## 26     26 0.03217344 0.2130227 0.02449452 0.0015829614 0.05476731 0.0009515173
## 27     27 0.03188007 0.2287608 0.02425934 0.0014848783 0.01808904 0.0007599195
## 28     28 0.03217137 0.2131521 0.02448780 0.0015829446 0.05470829 0.0009178775
## 29     29 0.03188794 0.2284677 0.02426722 0.0014998172 0.01848965 0.0007526385
## 30     30 0.03243517 0.1990803 0.02469611 0.0013719641 0.06501597 0.0009246369
## 31     31 0.03220993 0.2118676 0.02453475 0.0017269028 0.05629770 0.0010629551
## 32     32 0.03221820 0.2123366 0.02455062 0.0019162673 0.04877430 0.0011549851
## 33     33 0.03249936 0.1983642 0.02475349 0.0023659979 0.06942011 0.0015023842
## 34     34 0.03191266 0.2274897 0.02425482 0.0015268046 0.01876441 0.0007909804
## 35     35 0.03247522 0.1987429 0.02471692 0.0022093651 0.07299051 0.0014521814
## 36     36 0.03190813 0.2277650 0.02426104 0.0015560082 0.01982945 0.0008052846
## 37     37 0.03223258 0.2099059 0.02451746 0.0009485878 0.04410611 0.0006025639
## 38     38 0.03227727 0.2089095 0.02458393 0.0017346778 0.05471820 0.0010553270
## 39     39 0.03197215 0.2248803 0.02431971 0.0015393836 0.02034388 0.0008066505
## 40     40 0.03233838 0.2067715 0.02468061 0.0020323221 0.05609562 0.0014146263
## 41     41 0.03229553 0.2090460 0.02462658 0.0019364973 0.04922061 0.0011681077
## 42     42 0.03221976 0.2121239 0.02453166 0.0018615778 0.05603266 0.0011284147
## 43     43 0.03227746 0.2080250 0.02457319 0.0009367262 0.04436888 0.0006109973
## 44     44 0.03233589 0.2071655 0.02462639 0.0020238901 0.05512917 0.0013529163
## 45     45 0.03199435 0.2240801 0.02434657 0.0015484784 0.02213947 0.0008189863
## 46     46 0.03266134 0.1905784 0.02495662 0.0020038498 0.06568574 0.0012761348
## 47     47 0.03226006 0.2107893 0.02455824 0.0020741007 0.05647414 0.0012908067
## 48     48 0.03198958 0.2243218 0.02434109 0.0015288195 0.02152203 0.0008074663
## 49     49 0.03233186 0.2074798 0.02462514 0.0019863027 0.05413390 0.0013386773
## 50     50 0.03201448 0.2232140 0.02435494 0.0015148530 0.01989195 0.0007880686
## 51     51 0.03268682 0.1901362 0.02487722 0.0023688705 0.07026229 0.0015938838
## 52     52 0.03231908 0.2061982 0.02459253 0.0008835594 0.04384359 0.0005776377
## 53     53 0.03234391 0.2070537 0.02465347 0.0018805277 0.04783756 0.0011204084
## 54     54 0.03261395 0.1928685 0.02483925 0.0020486297 0.06696728 0.0014402877
## 55     55 0.03264168 0.1919794 0.02482033 0.0022367197 0.07176247 0.0014171764
## 56     56 0.03328563 0.1586380 0.02541356 0.0021447841 0.07691881 0.0015480125
## 57     57 0.03234703 0.2052065 0.02460237 0.0008960028 0.04248013 0.0005467922
## 58     58 0.03265249 0.1914349 0.02482248 0.0021227585 0.06782793 0.0013556656
## 59     59 0.03240856 0.2043786 0.02462767 0.0020605298 0.05460643 0.0012207046
## 60     60 0.03241490 0.2040246 0.02468663 0.0020083442 0.05361741 0.0013458322
## 61     61 0.03241288 0.2041966 0.02462990 0.0020497943 0.05386612 0.0012167188
## 62     62 0.03209531 0.2198515 0.02440774 0.0015328253 0.02010476 0.0007709214
## 63     63 0.03236620 0.2051305 0.02466267 0.0016360823 0.05500220 0.0009714223
## 64     64 0.03241423 0.2041815 0.02464620 0.0020398156 0.05438771 0.0011989078
## 65     65 0.03211335 0.2190882 0.02442752 0.0015070121 0.01881422 0.0007424970
## 66     66 0.03269445 0.1898503 0.02492431 0.0020799671 0.06611052 0.0012782085
## 67     67 0.03271559 0.1881345 0.02491342 0.0020475547 0.07020799 0.0012774070
## 68     68 0.03241369 0.2024513 0.02464596 0.0008820827 0.04031829 0.0004747953
## 69     69 0.03273508 0.1876800 0.02492596 0.0020492979 0.06709896 0.0014293520
## 70     70 0.03214723 0.2177466 0.02444997 0.0015039991 0.01910111 0.0007404796
## 71     71 0.03214959 0.2176219 0.02444372 0.0015071038 0.01898653 0.0007453285
## 72     72 0.03248193 0.2012172 0.02474384 0.0019530916 0.05195198 0.0012887492
## 73     73 0.03216115 0.2171176 0.02446895 0.0014974324 0.01852651 0.0007368656
## 74     74 0.03244606 0.2029449 0.02470512 0.0020231430 0.05377510 0.0012300466
## 75     75 0.03308808 0.1711649 0.02528696 0.0025076047 0.07458936 0.0016675343
## 76     76 0.03340790 0.1541890 0.02551801 0.0024169431 0.08275223 0.0016418401
## 77     77 0.03249477 0.2007505 0.02472838 0.0019946103 0.05304248 0.0011759759
## 78     78 0.03245384 0.2018245 0.02471200 0.0016330662 0.05024040 0.0009798261
## 79     79 0.03278900 0.1843157 0.02495524 0.0015421819 0.06008204 0.0009814597
## 80     80 0.03252634 0.1992337 0.02479449 0.0019305764 0.05170879 0.0012490100
## 81     81 0.03272850 0.1885874 0.02495296 0.0022231792 0.07086229 0.0014162099
## 82     82 0.03314735 0.1678891 0.02526869 0.0023213656 0.07577580 0.0015360648
## 83     83 0.03248695 0.2001288 0.02476418 0.0016207435 0.05547451 0.0009533852
## 84     84 0.03311718 0.1675490 0.02522765 0.0015875849 0.06863203 0.0010390800
## 85     85 0.03254903 0.1980546 0.02482389 0.0018996684 0.04945615 0.0012660742
## 86     86 0.03249884 0.1989080 0.02474991 0.0008836573 0.03972701 0.0004353838
## 87     87 0.03303314 0.1705538 0.02519277 0.0011200152 0.07237907 0.0008844522
## 88     88 0.03319987 0.1660270 0.02537710 0.0023414220 0.06823109 0.0016162505
## 89     89 0.03336531 0.1541739 0.02549540 0.0016329833 0.08051427 0.0011603990
## 90     90 0.03226808 0.2125706 0.02455784 0.0014628523 0.01855133 0.0007125177
## 91     91 0.03365576 0.1404118 0.02570860 0.0021150933 0.07768176 0.0015510349
## 92     92 0.03256623 0.1969279 0.02480390 0.0016547888 0.05258491 0.0009805765
## 93     93 0.03251401 0.2001550 0.02476410 0.0020270301 0.05470975 0.0012510205
## 94     94 0.03310519 0.1699529 0.02521803 0.0021893774 0.07889647 0.0014987772
## 95     95 0.03314994 0.1685239 0.02529370 0.0025429108 0.07574922 0.0017437697
## 96     96 0.03226132 0.2129982 0.02453195 0.0014716093 0.01906048 0.0006972590
## 97     97 0.03282911 0.1837438 0.02503513 0.0021003485 0.06899595 0.0013525585
## 98     98 0.03280902 0.1830274 0.02499058 0.0010976988 0.05766041 0.0007215989
## 99     99 0.03225569 0.2132426 0.02453825 0.0014636229 0.01864073 0.0006926260
## 100   100 0.03252677 0.1990049 0.02475380 0.0016626218 0.05223040 0.0010098968
## 101   101 0.03282695 0.1842727 0.02496953 0.0020507737 0.06699193 0.0013010182
## 102   102 0.03286864 0.1819391 0.02507897 0.0019883071 0.06608629 0.0014232390
## 103   103 0.03257685 0.1968387 0.02482673 0.0016087142 0.05712864 0.0009676349
## 104   104 0.03314723 0.1673829 0.02527047 0.0017594057 0.06303653 0.0010954956
## 105   105 0.03263572 0.1949319 0.02486286 0.0019403455 0.05277805 0.0012411615
## 106   106 0.03318712 0.1655025 0.02534671 0.0020400224 0.07554857 0.0014832433
## 107   107 0.03228420 0.2121080 0.02457949 0.0014784549 0.01956456 0.0007104400
## 108   108 0.03347720 0.1509767 0.02553144 0.0020898970 0.08574186 0.0014122523
## 109   109 0.03283119 0.1846365 0.02503756 0.0022007575 0.07094850 0.0013889416
## 110   110 0.03314688 0.1668267 0.02527694 0.0015909117 0.06901615 0.0011901243
## 111   111 0.03261703 0.1950725 0.02485438 0.0016515192 0.05322632 0.0009776952
## 112   112 0.03260833 0.1957256 0.02486679 0.0016295716 0.05835949 0.0009993505
## 113   113 0.03263560 0.1943149 0.02487240 0.0016480981 0.05301725 0.0009841213
## 114   114 0.03268108 0.1931221 0.02491581 0.0019375400 0.05292639 0.0012607205
## 115   115 0.03296391 0.1790868 0.02514677 0.0021655284 0.06137569 0.0013072645
## 116   116 0.03292241 0.1810188 0.02506754 0.0021328636 0.06843435 0.0012959052
## 117   117 0.03258909 0.1972866 0.02484228 0.0019923554 0.05432273 0.0012332739
## 118   118 0.03321022 0.1648566 0.02532890 0.0016997897 0.07034424 0.0011988519
## 119   119 0.03290497 0.1811765 0.02505279 0.0019058632 0.07163510 0.0012030192
## 120   120 0.03285779 0.1837389 0.02513362 0.0018176674 0.05243753 0.0013166321
## 121   121 0.03257005 0.1976285 0.02484387 0.0015227162 0.04797309 0.0008885470
## 122   122 0.03289550 0.1827860 0.02516803 0.0019791082 0.05950836 0.0014188178
## 123   123 0.03257914 0.1983400 0.02481751 0.0017550072 0.03886913 0.0010106471
## 124   124 0.03257291 0.1978677 0.02483419 0.0015185099 0.03863327 0.0008503827
## 125   125 0.03306116 0.1748334 0.02524364 0.0020792232 0.05048285 0.0013032221
## 126   126 0.03236703 0.2087673 0.02465019 0.0014508566 0.01977178 0.0007113957
## 127   127 0.03284884 0.1848806 0.02506451 0.0021216151 0.05790211 0.0013697874
## 128   128 0.03323604 0.1640946 0.02535457 0.0018453954 0.07106640 0.0012490456
## 129   129 0.03236539 0.2089080 0.02464489 0.0014524989 0.02001379 0.0007081067
## 130   130 0.03236819 0.2088150 0.02465015 0.0014580603 0.02033253 0.0007132284
## 131   131 0.03274153 0.1889321 0.02497454 0.0012908334 0.03285839 0.0006166551
## 132   132 0.03275108 0.1890961 0.02495133 0.0015305834 0.04001633 0.0009375302
## 133   133 0.03252298 0.2004755 0.02479051 0.0014835073 0.03123479 0.0007757762
## 134   134 0.03295910 0.1791839 0.02514813 0.0019272954 0.05354532 0.0012182543
## 135   135 0.03260027 0.1972983 0.02481917 0.0016125689 0.04529574 0.0009356891
## 136   136 0.03249634 0.2011953 0.02474905 0.0011595581 0.02253492 0.0005395138
## 137   137 0.03256254 0.1993243 0.02481833 0.0016273370 0.04056024 0.0008999213
## 138   138 0.03293877 0.1797129 0.02511367 0.0017133619 0.06126315 0.0010534673
## 139   139 0.03250168 0.2009887 0.02475038 0.0011491195 0.02266186 0.0005306931
## 140   140 0.03251613 0.2011584 0.02476675 0.0017465890 0.03843589 0.0009616847
## 141   141 0.03260613 0.1972270 0.02482052 0.0016252138 0.04597415 0.0009429437
## 142   142 0.03236819 0.2089163 0.02465593 0.0014781186 0.02080564 0.0007316022
## 143   143 0.03236240 0.2091646 0.02464914 0.0014781540 0.02086623 0.0007347833
## 144   144 0.03273593 0.1909507 0.02497309 0.0017327125 0.04199436 0.0009254589
## 145   145 0.03265612 0.1932054 0.02489720 0.0017628112 0.04942052 0.0010044075
## 146   146 0.03236841 0.2089066 0.02465984 0.0014939256 0.02058848 0.0007464648
## 147   147 0.03294125 0.1801660 0.02510840 0.0018059255 0.05429035 0.0011421109
## 148   148 0.03260575 0.1973044 0.02490483 0.0017151420 0.03455884 0.0011256830
## 149   149 0.03263429 0.1963672 0.02487551 0.0018336360 0.04293736 0.0011208681
## 150   150 0.03253826 0.2006433 0.02480000 0.0017135530 0.03375773 0.0009797850
## 151   151 0.03261040 0.1971908 0.02483814 0.0016507077 0.04644406 0.0009758165
## 152   152 0.03260111 0.1975502 0.02489885 0.0017122506 0.03452387 0.0011218722
## 153   153 0.03272995 0.1906129 0.02495581 0.0016657374 0.04752930 0.0009643095
## 154   154 0.03250622 0.2008677 0.02476304 0.0011464526 0.02283066 0.0005172392
## 155   155 0.03236720 0.2090281 0.02465408 0.0014989413 0.02117697 0.0007495190
## 156   156 0.03261125 0.1971338 0.02484246 0.0016547486 0.04661932 0.0009897353
## 157   157 0.03270935 0.1915623 0.02493874 0.0017346148 0.04196933 0.0010350970
## 158   158 0.03277594 0.1883870 0.02499709 0.0018442832 0.05263073 0.0011521600
## 159   159 0.03277036 0.1892381 0.02497073 0.0018283752 0.05152345 0.0011530466
## 160   160 0.03250114 0.2010583 0.02476125 0.0011450999 0.02251932 0.0005189684
## 161   161 0.03260474 0.1974062 0.02483795 0.0016508997 0.04654346 0.0009918247
## 162   162 0.03254400 0.1997109 0.02480357 0.0015401874 0.03490539 0.0008431535
## 163   163 0.03276857 0.1890009 0.02503862 0.0019911611 0.04823018 0.0013250090
## 164   164 0.03267743 0.1921776 0.02490670 0.0015349745 0.04096885 0.0008719480
## 165   165 0.03269353 0.1919742 0.02492901 0.0017308455 0.04667546 0.0010107058
## 166   166 0.03283203 0.1862188 0.02508620 0.0018151716 0.05127240 0.0012758771
## 167   167 0.03254602 0.2000107 0.02480654 0.0018381431 0.04248665 0.0010481664
## 168   168 0.03267807 0.1917980 0.02491901 0.0011568664 0.03305417 0.0005779621
## 169   169 0.03253834 0.1999605 0.02480121 0.0015293145 0.03444718 0.0008295579
## 170   170 0.03236686 0.2090271 0.02466285 0.0014905458 0.02034129 0.0007556632
## 171   171 0.03236412 0.2091623 0.02466255 0.0014899793 0.02042908 0.0007526824
## 172   172 0.03254172 0.2004715 0.02482979 0.0016256814 0.02778183 0.0008320188
## 173   173 0.03253476 0.1999953 0.02480750 0.0015312456 0.04096353 0.0008199110
## 174   174 0.03260775 0.1973623 0.02490991 0.0017243916 0.03528396 0.0011381611
## 175   175 0.03330178 0.1609723 0.02546237 0.0015007767 0.04791025 0.0008611774
## 176   176 0.03237733 0.2086506 0.02467762 0.0014941193 0.02026212 0.0007549749
## 177   177 0.03252882 0.1999239 0.02480378 0.0011045865 0.02296427 0.0004836977
## 178   178 0.03289541 0.1823825 0.02511881 0.0019014543 0.04770800 0.0010696172
## 179   179 0.03254524 0.2001675 0.02481130 0.0018122531 0.04072774 0.0010217248
## 180   180 0.03276087 0.1889033 0.02498130 0.0016367341 0.05457661 0.0009902407
## 181   181 0.03260715 0.1973914 0.02490536 0.0017155085 0.03454166 0.0011244690
## 182   182 0.03255419 0.2000182 0.02482842 0.0017207676 0.03372765 0.0010040404
## 183   183 0.03299272 0.1780147 0.02517693 0.0020850627 0.06108021 0.0013020661
## 184   184 0.03255074 0.2001229 0.02483816 0.0016193370 0.02734372 0.0008199104
## 185   185 0.03254053 0.1993728 0.02480752 0.0010848083 0.02326984 0.0004681806
## 186   186 0.03254357 0.1992441 0.02480862 0.0010840425 0.02341089 0.0004670829
## 187   187 0.03238687 0.2082370 0.02467856 0.0014833531 0.01946381 0.0007377509
## 188   188 0.03254527 0.1991925 0.02480677 0.0010877217 0.02319290 0.0004717689
## 189   189 0.03239142 0.2080422 0.02468135 0.0014833719 0.01929048 0.0007396397
## 190   190 0.03257397 0.1991867 0.02483585 0.0017121754 0.03318199 0.0009879432
## 191   191 0.03273931 0.1908163 0.02498809 0.0019085687 0.04336538 0.0010663324
## 192   192 0.03256036 0.1989701 0.02482273 0.0015104231 0.03989163 0.0008026181
## 193   193 0.03239613 0.2078628 0.02468274 0.0014786953 0.01919531 0.0007367794
## 194   194 0.03280194 0.1883672 0.02501327 0.0019719812 0.05461423 0.0011699536
## 195   195 0.03239522 0.2079060 0.02468292 0.0014799569 0.01911824 0.0007385234
## 196   196 0.03239581 0.2078831 0.02468360 0.0014806961 0.01914555 0.0007407453
## 197   197 0.03295205 0.1786101 0.02519601 0.0013433211 0.04428285 0.0009536884
## 198   198 0.03239846 0.2077625 0.02468544 0.0014810663 0.01906137 0.0007402497
## 199   199 0.03257201 0.1992179 0.02484568 0.0016162869 0.02742018 0.0008206673
## 200   200 0.03262961 0.1965477 0.02492122 0.0017145543 0.03444378 0.0011418020
## 201   201 0.03239598 0.2078783 0.02468274 0.0014824224 0.01914546 0.0007402610
## 202   202 0.03239485 0.2079343 0.02468363 0.0014824109 0.01907671 0.0007401114
## 203   203 0.03278960 0.1874366 0.02504980 0.0013661704 0.03433703 0.0009578178
## 204   204 0.03281049 0.1877476 0.02508267 0.0018179708 0.03741249 0.0011611004
## 205   205 0.03266470 0.1953006 0.02490038 0.0018206092 0.04177308 0.0011152750
## 206   206 0.03239920 0.2077747 0.02468324 0.0014840916 0.01913154 0.0007416637
## 207   207 0.03256924 0.1986944 0.02482736 0.0015156865 0.04014717 0.0008099810
## 208   208 0.03239768 0.2078401 0.02468274 0.0014836879 0.01922151 0.0007412517
## 209   209 0.03239804 0.2078230 0.02468236 0.0014822092 0.01916628 0.0007398635
## 210   210 0.03239803 0.2078255 0.02468221 0.0014818460 0.01917665 0.0007388410
## 211   211 0.03239735 0.2078514 0.02468182 0.0014816975 0.01915682 0.0007380946
## 212   212 0.03239737 0.2078444 0.02468275 0.0014808361 0.01913159 0.0007377400
## 213   213 0.03239639 0.2078954 0.02468304 0.0014800765 0.01909881 0.0007370320
## 214   214 0.03239698 0.2078692 0.02468424 0.0014798088 0.01908598 0.0007365311
## 215   215 0.03239708 0.2078651 0.02468436 0.0014809105 0.01910152 0.0007371423
## 216   216 0.03239731 0.2078526 0.02468586 0.0014807690 0.01909326 0.0007370225
## 217   217 0.03239844 0.2078016 0.02468647 0.0014805084 0.01910005 0.0007367713
## 218   218 0.03262905 0.1966180 0.02487031 0.0016208046 0.04406898 0.0009807548
## 219   219 0.03239870 0.2077968 0.02468589 0.0014812351 0.01909185 0.0007371097
## 220   220 0.03239777 0.2078409 0.02468520 0.0014813970 0.01909433 0.0007383870
## 221   221 0.03239771 0.2078455 0.02468459 0.0014809400 0.01911086 0.0007386547
## 222   222 0.03263309 0.1971244 0.02486697 0.0017017880 0.04359752 0.0009492934
## 223   223 0.03280876 0.1872858 0.02506327 0.0017358566 0.04758575 0.0011611036
## 224   224 0.03267567 0.1949703 0.02491257 0.0018440887 0.04293876 0.0011446470
## 225   225 0.03239558 0.2079363 0.02468282 0.0014818300 0.01916216 0.0007403572
## 226   226 0.03263086 0.1972184 0.02486748 0.0017016250 0.04350346 0.0009543761
## 227   227 0.03239493 0.2079645 0.02468266 0.0014818831 0.01918029 0.0007405759
## 228   228 0.03239388 0.2080087 0.02468201 0.0014811758 0.01915911 0.0007403683
## 229   229 0.03239372 0.2080174 0.02468183 0.0014808696 0.01915881 0.0007401023
## 230   230 0.03239384 0.2080118 0.02468183 0.0014809846 0.01915371 0.0007400824
## 231   231 0.03264131 0.1962663 0.02492477 0.0017404047 0.03591915 0.0011571189
## 232   232 0.03292902 0.1811937 0.02513348 0.0018103384 0.04779928 0.0010164070
## 233   233 0.03263314 0.1971515 0.02487064 0.0017070594 0.04386865 0.0009622500
## 234   234 0.03260416 0.1975101 0.02485731 0.0015556930 0.03742956 0.0008672605
## 235   235 0.03310138 0.1744482 0.02529309 0.0021316255 0.06161872 0.0014784938
## 236   236 0.03239430 0.2079919 0.02468277 0.0014813823 0.01914093 0.0007400643
## 237   237 0.03298648 0.1792530 0.02515229 0.0018715158 0.05787469 0.0011194834
## 238   238 0.03333535 0.1634123 0.02545920 0.0020714256 0.06235155 0.0014270818
## 239   239 0.03307479 0.1748240 0.02525088 0.0018667650 0.05899729 0.0012211908
## 240   240 0.03239445 0.2079861 0.02468299 0.0014813303 0.01914509 0.0007400672
## [1] "Best Model"
##    nvmax
## 19    19

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  1.988237e+00  1.980581e+00  1.995894e+00
## x4          -4.321837e-05 -6.068012e-05 -2.575661e-05
## x7           1.118167e-02  9.953279e-03  1.241005e-02
## x8           4.604115e-04  1.728983e-04  7.479248e-04
## x9           3.144328e-03  2.500333e-03  3.788324e-03
## x10          1.051023e-03  4.549996e-04  1.647046e-03
## x16          1.019789e-03  6.076877e-04  1.431891e-03
## x17          1.632689e-03  1.001442e-03  2.263935e-03
## x21          1.432126e-04  6.151940e-05  2.249057e-04
## stat13      -7.132321e-04 -1.189964e-03 -2.364999e-04
## stat14      -8.569270e-04 -1.332110e-03 -3.817434e-04
## stat23       6.971062e-04  2.190595e-04  1.175153e-03
## stat24      -7.739189e-04 -1.254354e-03 -2.934836e-04
## stat60       5.992247e-04  1.187520e-04  1.079697e-03
## stat91      -6.246528e-04 -1.101914e-03 -1.473917e-04
## stat98       3.559182e-03  3.087007e-03  4.031356e-03
## stat110     -3.508962e-03 -3.988726e-03 -3.029198e-03
## stat149     -7.085899e-04 -1.191292e-03 -2.258873e-04
## stat195      6.602235e-04  1.830305e-04  1.137416e-03
## x18.sqrt     2.617529e-02  2.433567e-02  2.801490e-02

Test

if (algo.stepwise.caret == TRUE){
  test.model(model.stepwise, data.test
             ,method = 'leapSeq',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
  
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.038   2.084   2.097   2.097   2.110   2.147 
## [1] "leapSeq  Test MSE: 0.000964626165422909"

LASSO with CV

Train

if (algo.LASSO.caret == TRUE){
  set.seed(1)
  tune.grid= expand.grid(alpha = 1,lambda = 10^seq(from=-4,to=-2,length=100))
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "glmnet"
                                   ,subopt = 'LASSO'
                                   ,tune.grid = tune.grid
                                   ,feature.names = feature.names)
  model.LASSO.caret = returned$model
}
## Aggregating results
## Selecting tuning parameters
## Fitting alpha = 1, lambda = 0.000586 on full training set
## glmnet 
## 
## 5584 samples
##  240 predictor
## 
## No pre-processing
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 5026, 5026, 5026, 5025, 5025, 5026, ... 
## Resampling results across tuning parameters:
## 
##   lambda        RMSE        Rsquared   MAE       
##   0.0001000000  0.03215804  0.2164349  0.02449844
##   0.0001047616  0.03214874  0.2167920  0.02449116
##   0.0001097499  0.03213938  0.2171527  0.02448388
##   0.0001149757  0.03212988  0.2175205  0.02447663
##   0.0001204504  0.03212035  0.2178903  0.02446938
##   0.0001261857  0.03211070  0.2182669  0.02446208
##   0.0001321941  0.03210109  0.2186420  0.02445482
##   0.0001384886  0.03209139  0.2190236  0.02444757
##   0.0001450829  0.03208149  0.2194153  0.02444018
##   0.0001519911  0.03207142  0.2198169  0.02443264
##   0.0001592283  0.03206105  0.2202347  0.02442481
##   0.0001668101  0.03205047  0.2206647  0.02441688
##   0.0001747528  0.03203962  0.2211103  0.02440878
##   0.0001830738  0.03202838  0.2215766  0.02440057
##   0.0001917910  0.03201673  0.2220656  0.02439224
##   0.0002009233  0.03200502  0.2225614  0.02438379
##   0.0002104904  0.03199335  0.2230593  0.02437539
##   0.0002205131  0.03198156  0.2235670  0.02436703
##   0.0002310130  0.03196967  0.2240848  0.02435842
##   0.0002420128  0.03195756  0.2246184  0.02434957
##   0.0002535364  0.03194513  0.2251750  0.02434048
##   0.0002656088  0.03193273  0.2257386  0.02433110
##   0.0002782559  0.03192028  0.2263141  0.02432158
##   0.0002915053  0.03190814  0.2268842  0.02431260
##   0.0003053856  0.03189630  0.2274506  0.02430390
##   0.0003199267  0.03188489  0.2280070  0.02429555
##   0.0003351603  0.03187380  0.2285594  0.02428764
##   0.0003511192  0.03186320  0.2290998  0.02428043
##   0.0003678380  0.03185278  0.2296454  0.02427381
##   0.0003853529  0.03184311  0.2301671  0.02426823
##   0.0004037017  0.03183444  0.2306512  0.02426375
##   0.0004229243  0.03182653  0.2311122  0.02425995
##   0.0004430621  0.03181981  0.2315270  0.02425688
##   0.0004641589  0.03181423  0.2318992  0.02425485
##   0.0004862602  0.03181026  0.2322029  0.02425440
##   0.0005094138  0.03180724  0.2324736  0.02425463
##   0.0005336699  0.03180461  0.2327404  0.02425556
##   0.0005590810  0.03180319  0.2329623  0.02425768
##   0.0005857021  0.03180311  0.2331309  0.02426118
##   0.0006135907  0.03180480  0.2332235  0.02426607
##   0.0006428073  0.03180848  0.2332267  0.02427264
##   0.0006734151  0.03181436  0.2331305  0.02428085
##   0.0007054802  0.03182219  0.2329455  0.02429051
##   0.0007390722  0.03183146  0.2327021  0.02430157
##   0.0007742637  0.03184234  0.2323931  0.02431452
##   0.0008111308  0.03185514  0.2320005  0.02432954
##   0.0008497534  0.03186933  0.2315576  0.02434591
##   0.0008902151  0.03188499  0.2310533  0.02436285
##   0.0009326033  0.03190191  0.2305078  0.02438090
##   0.0009770100  0.03192043  0.2298960  0.02440035
##   0.0010235310  0.03194099  0.2291965  0.02442123
##   0.0010722672  0.03196251  0.2284659  0.02444298
##   0.0011233240  0.03198469  0.2277235  0.02446578
##   0.0011768120  0.03200808  0.2269409  0.02448951
##   0.0012328467  0.03203153  0.2261887  0.02451402
##   0.0012915497  0.03205558  0.2254439  0.02453871
##   0.0013530478  0.03207974  0.2247374  0.02456331
##   0.0014174742  0.03210533  0.2239911  0.02458900
##   0.0014849683  0.03213112  0.2232827  0.02461569
##   0.0015556761  0.03215874  0.2225214  0.02464394
##   0.0016297508  0.03218761  0.2217497  0.02467418
##   0.0017073526  0.03221928  0.2208755  0.02470733
##   0.0017886495  0.03225411  0.2198844  0.02474290
##   0.0018738174  0.03229075  0.2188555  0.02478010
##   0.0019630407  0.03232646  0.2179734  0.02481632
##   0.0020565123  0.03236264  0.2171480  0.02485289
##   0.0021544347  0.03239927  0.2163996  0.02488961
##   0.0022570197  0.03243707  0.2156840  0.02492682
##   0.0023644894  0.03247523  0.2150855  0.02496485
##   0.0024770764  0.03251701  0.2143976  0.02500584
##   0.0025950242  0.03256298  0.2135952  0.02505007
##   0.0027185882  0.03261336  0.2126639  0.02509786
##   0.0028480359  0.03266857  0.2115791  0.02514890
##   0.0029836472  0.03272906  0.2103111  0.02520378
##   0.0031257158  0.03279533  0.2088232  0.02526312
##   0.0032745492  0.03286792  0.2070704  0.02532682
##   0.0034304693  0.03294741  0.2049970  0.02539578
##   0.0035938137  0.03303429  0.2025462  0.02546936
##   0.0037649358  0.03312896  0.1996585  0.02554823
##   0.0039442061  0.03322531  0.1968079  0.02562951
##   0.0041320124  0.03332001  0.1943482  0.02571065
##   0.0043287613  0.03342115  0.1916081  0.02579559
##   0.0045348785  0.03352890  0.1885618  0.02588375
##   0.0047508102  0.03364676  0.1848099  0.02597805
##   0.0049770236  0.03377563  0.1801565  0.02607963
##   0.0052140083  0.03391651  0.1743468  0.02618952
##   0.0054622772  0.03407046  0.1670531  0.02631008
##   0.0057223677  0.03423580  0.1581620  0.02643827
##   0.0059948425  0.03441036  0.1476446  0.02657199
##   0.0062802914  0.03457608  0.1373952  0.02669670
##   0.0065793322  0.03470910  0.1305239  0.02678984
##   0.0068926121  0.03484059  0.1236362  0.02688165
##   0.0072208090  0.03496992  0.1166880  0.02697035
##   0.0075646333  0.03509152  0.1104593  0.02705348
##   0.0079248290  0.03518700  0.1080195  0.02711703
##   0.0083021757  0.03528151  0.1067604  0.02717917
##   0.0086974900  0.03537645  0.1067604  0.02724148
##   0.0091116276  0.03548036  0.1067604  0.02731132
##   0.0095454846  0.03559406  0.1067604  0.02738858
##   0.0100000000  0.03571844  0.1067604  0.02747425
## 
## Tuning parameter 'alpha' was held constant at a value of 1
## RMSE was used to select the optimal model using the smallest value.
## The final values used for the model were alpha = 1 and lambda = 0.0005857021.

##    alpha       lambda
## 39     1 0.0005857021
##     alpha       lambda       RMSE  Rsquared        MAE      RMSESD RsquaredSD        MAESD
## 1       1 0.0001000000 0.03215804 0.2164349 0.02449844 0.001493272 0.01958722 0.0007431067
## 2       1 0.0001047616 0.03214874 0.2167920 0.02449116 0.001493827 0.01962500 0.0007435796
## 3       1 0.0001097499 0.03213938 0.2171527 0.02448388 0.001494475 0.01966449 0.0007440280
## 4       1 0.0001149757 0.03212988 0.2175205 0.02447663 0.001495214 0.01970062 0.0007445821
## 5       1 0.0001204504 0.03212035 0.2178903 0.02446938 0.001495785 0.01972312 0.0007450265
## 6       1 0.0001261857 0.03211070 0.2182669 0.02446208 0.001496372 0.01974137 0.0007454520
## 7       1 0.0001321941 0.03210109 0.2186420 0.02445482 0.001496757 0.01974294 0.0007456979
## 8       1 0.0001384886 0.03209139 0.2190236 0.02444757 0.001497208 0.01974052 0.0007460360
## 9       1 0.0001450829 0.03208149 0.2194153 0.02444018 0.001497608 0.01973145 0.0007463005
## 10      1 0.0001519911 0.03207142 0.2198169 0.02443264 0.001497966 0.01971767 0.0007465349
## 11      1 0.0001592283 0.03206105 0.2202347 0.02442481 0.001498253 0.01969970 0.0007468179
## 12      1 0.0001668101 0.03205047 0.2206647 0.02441688 0.001498494 0.01967708 0.0007472741
## 13      1 0.0001747528 0.03203962 0.2211103 0.02440878 0.001498619 0.01964397 0.0007475700
## 14      1 0.0001830738 0.03202838 0.2215766 0.02440057 0.001498607 0.01960783 0.0007478619
## 15      1 0.0001917910 0.03201673 0.2220656 0.02439224 0.001498369 0.01955627 0.0007478979
## 16      1 0.0002009233 0.03200502 0.2225614 0.02438379 0.001498040 0.01949511 0.0007478911
## 17      1 0.0002104904 0.03199335 0.2230593 0.02437539 0.001497477 0.01941486 0.0007473658
## 18      1 0.0002205131 0.03198156 0.2235670 0.02436703 0.001496858 0.01932317 0.0007467860
## 19      1 0.0002310130 0.03196967 0.2240848 0.02435842 0.001496392 0.01922466 0.0007463960
## 20      1 0.0002420128 0.03195756 0.2246184 0.02434957 0.001495890 0.01912789 0.0007461066
## 21      1 0.0002535364 0.03194513 0.2251750 0.02434048 0.001495891 0.01904142 0.0007463391
## 22      1 0.0002656088 0.03193273 0.2257386 0.02433110 0.001495932 0.01895554 0.0007466610
## 23      1 0.0002782559 0.03192028 0.2263141 0.02432158 0.001496519 0.01887895 0.0007477827
## 24      1 0.0002915053 0.03190814 0.2268842 0.02431260 0.001497226 0.01881371 0.0007482578
## 25      1 0.0003053856 0.03189630 0.2274506 0.02430390 0.001498459 0.01877164 0.0007486831
## 26      1 0.0003199267 0.03188489 0.2280070 0.02429555 0.001499833 0.01872583 0.0007492348
## 27      1 0.0003351603 0.03187380 0.2285594 0.02428764 0.001501595 0.01868964 0.0007508707
## 28      1 0.0003511192 0.03186320 0.2290998 0.02428043 0.001503566 0.01865180 0.0007525696
## 29      1 0.0003678380 0.03185278 0.2296454 0.02427381 0.001506062 0.01863585 0.0007549367
## 30      1 0.0003853529 0.03184311 0.2301671 0.02426823 0.001508638 0.01862432 0.0007578233
## 31      1 0.0004037017 0.03183444 0.2306512 0.02426375 0.001510708 0.01861314 0.0007609794
## 32      1 0.0004229243 0.03182653 0.2311122 0.02425995 0.001512653 0.01861422 0.0007645607
## 33      1 0.0004430621 0.03181981 0.2315270 0.02425688 0.001514072 0.01862889 0.0007680629
## 34      1 0.0004641589 0.03181423 0.2318992 0.02425485 0.001515647 0.01865947 0.0007714641
## 35      1 0.0004862602 0.03181026 0.2322029 0.02425440 0.001516214 0.01867652 0.0007734909
## 36      1 0.0005094138 0.03180724 0.2324736 0.02425463 0.001516677 0.01868645 0.0007752604
## 37      1 0.0005336699 0.03180461 0.2327404 0.02425556 0.001517423 0.01867717 0.0007775926
## 38      1 0.0005590810 0.03180319 0.2329623 0.02425768 0.001517844 0.01864640 0.0007793484
## 39      1 0.0005857021 0.03180311 0.2331309 0.02426118 0.001517459 0.01857451 0.0007801326
## 40      1 0.0006135907 0.03180480 0.2332235 0.02426607 0.001516979 0.01849960 0.0007809935
## 41      1 0.0006428073 0.03180848 0.2332267 0.02427264 0.001515922 0.01841537 0.0007816864
## 42      1 0.0006734151 0.03181436 0.2331305 0.02428085 0.001514820 0.01833596 0.0007824873
## 43      1 0.0007054802 0.03182219 0.2329455 0.02429051 0.001513904 0.01828733 0.0007837394
## 44      1 0.0007390722 0.03183146 0.2327021 0.02430157 0.001512828 0.01825401 0.0007851120
## 45      1 0.0007742637 0.03184234 0.2323931 0.02431452 0.001512059 0.01823988 0.0007862712
## 46      1 0.0008111308 0.03185514 0.2320005 0.02432954 0.001511327 0.01820602 0.0007878479
## 47      1 0.0008497534 0.03186933 0.2315576 0.02434591 0.001512635 0.01816656 0.0007904399
## 48      1 0.0008902151 0.03188499 0.2310533 0.02436285 0.001513748 0.01813249 0.0007923637
## 49      1 0.0009326033 0.03190191 0.2305078 0.02438090 0.001516009 0.01811319 0.0007945023
## 50      1 0.0009770100 0.03192043 0.2298960 0.02440035 0.001518500 0.01813136 0.0007964450
## 51      1 0.0010235310 0.03194099 0.2291965 0.02442123 0.001521364 0.01818427 0.0007987084
## 52      1 0.0010722672 0.03196251 0.2284659 0.02444298 0.001523788 0.01825883 0.0008009059
## 53      1 0.0011233240 0.03198469 0.2277235 0.02446578 0.001525541 0.01831575 0.0008029846
## 54      1 0.0011768120 0.03200808 0.2269409 0.02448951 0.001526760 0.01837685 0.0008047111
## 55      1 0.0012328467 0.03203153 0.2261887 0.02451402 0.001528683 0.01845832 0.0008071506
## 56      1 0.0012915497 0.03205558 0.2254439 0.02453871 0.001530168 0.01855947 0.0008089566
## 57      1 0.0013530478 0.03207974 0.2247374 0.02456331 0.001531771 0.01864116 0.0008109644
## 58      1 0.0014174742 0.03210533 0.2239911 0.02458900 0.001532966 0.01872079 0.0008122719
## 59      1 0.0014849683 0.03213112 0.2232827 0.02461569 0.001535155 0.01880462 0.0008138425
## 60      1 0.0015556761 0.03215874 0.2225214 0.02464394 0.001537315 0.01890154 0.0008150939
## 61      1 0.0016297508 0.03218761 0.2217497 0.02467418 0.001539663 0.01900827 0.0008161384
## 62      1 0.0017073526 0.03221928 0.2208755 0.02470733 0.001541888 0.01912972 0.0008174874
## 63      1 0.0017886495 0.03225411 0.2198844 0.02474290 0.001543660 0.01926723 0.0008188486
## 64      1 0.0018738174 0.03229075 0.2188555 0.02478010 0.001544274 0.01934503 0.0008196373
## 65      1 0.0019630407 0.03232646 0.2179734 0.02481632 0.001544575 0.01930739 0.0008210577
## 66      1 0.0020565123 0.03236264 0.2171480 0.02485289 0.001544175 0.01929685 0.0008224921
## 67      1 0.0021544347 0.03239927 0.2163996 0.02488961 0.001543875 0.01925664 0.0008237815
## 68      1 0.0022570197 0.03243707 0.2156840 0.02492682 0.001542468 0.01925839 0.0008236581
## 69      1 0.0023644894 0.03247523 0.2150855 0.02496485 0.001541906 0.01927099 0.0008236523
## 70      1 0.0024770764 0.03251701 0.2143976 0.02500584 0.001541068 0.01928877 0.0008233007
## 71      1 0.0025950242 0.03256298 0.2135952 0.02505007 0.001539811 0.01930256 0.0008220103
## 72      1 0.0027185882 0.03261336 0.2126639 0.02509786 0.001538413 0.01931816 0.0008209183
## 73      1 0.0028480359 0.03266857 0.2115791 0.02514890 0.001536864 0.01933577 0.0008200652
## 74      1 0.0029836472 0.03272906 0.2103111 0.02520378 0.001535148 0.01935559 0.0008188590
## 75      1 0.0031257158 0.03279533 0.2088232 0.02526312 0.001533253 0.01937776 0.0008180959
## 76      1 0.0032745492 0.03286792 0.2070704 0.02532682 0.001531165 0.01940237 0.0008177751
## 77      1 0.0034304693 0.03294741 0.2049970 0.02539578 0.001528867 0.01942934 0.0008175887
## 78      1 0.0035938137 0.03303429 0.2025462 0.02546936 0.001526360 0.01944644 0.0008175722
## 79      1 0.0037649358 0.03312896 0.1996585 0.02554823 0.001523658 0.01942640 0.0008171696
## 80      1 0.0039442061 0.03322531 0.1968079 0.02562951 0.001518427 0.01955030 0.0008163094
## 81      1 0.0041320124 0.03332001 0.1943482 0.02571065 0.001516272 0.01950704 0.0008168425
## 82      1 0.0043287613 0.03342115 0.1916081 0.02579559 0.001513503 0.01966783 0.0008169212
## 83      1 0.0045348785 0.03352890 0.1885618 0.02588375 0.001510841 0.01991419 0.0008160824
## 84      1 0.0047508102 0.03364676 0.1848099 0.02597805 0.001508013 0.02023385 0.0008142063
## 85      1 0.0049770236 0.03377563 0.1801565 0.02607963 0.001505002 0.02064006 0.0008119796
## 86      1 0.0052140083 0.03391651 0.1743468 0.02618952 0.001501804 0.02114330 0.0008096488
## 87      1 0.0054622772 0.03407046 0.1670531 0.02631008 0.001498416 0.02174500 0.0008056870
## 88      1 0.0057223677 0.03423580 0.1581620 0.02643827 0.001493965 0.02230550 0.0007991666
## 89      1 0.0059948425 0.03441036 0.1476446 0.02657199 0.001488598 0.02248763 0.0007916416
## 90      1 0.0062802914 0.03457608 0.1373952 0.02669670 0.001476175 0.02328057 0.0007788764
## 91      1 0.0065793322 0.03470910 0.1305239 0.02678984 0.001475897 0.02129314 0.0007809256
## 92      1 0.0068926121 0.03484059 0.1236362 0.02688165 0.001471250 0.02097446 0.0007788608
## 93      1 0.0072208090 0.03496992 0.1166880 0.02697035 0.001471060 0.02035973 0.0007811866
## 94      1 0.0075646333 0.03509152 0.1104593 0.02705348 0.001466699 0.02104359 0.0007799671
## 95      1 0.0079248290 0.03518700 0.1080195 0.02711703 0.001478183 0.01929555 0.0007905514
## 96      1 0.0083021757 0.03528151 0.1067604 0.02717917 0.001486823 0.01920345 0.0007978703
## 97      1 0.0086974900 0.03537645 0.1067604 0.02724148 0.001493252 0.01920345 0.0008043945
## 98      1 0.0091116276 0.03548036 0.1067604 0.02731132 0.001500120 0.01920345 0.0008106162
## 99      1 0.0095454846 0.03559406 0.1067604 0.02738858 0.001507461 0.01920345 0.0008177135
## 100     1 0.0100000000 0.03571844 0.1067604 0.02747425 0.001515310 0.01920345 0.0008268885

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients"
##                model.coef
## (Intercept)  2.000645e+00
## x4          -3.160800e-05
## x7           1.031342e-02
## x8           2.547054e-04
## x9           2.691052e-03
## x10          6.350427e-04
## x11          2.061496e+04
## x14         -2.482849e-04
## x16          7.220313e-04
## x17          1.205529e-03
## x21          8.689675e-05
## stat3        1.533469e-04
## stat4       -7.250160e-05
## stat8        2.043020e-05
## stat13      -4.132607e-04
## stat14      -5.233028e-04
## stat18      -2.930538e-06
## stat20      -1.328948e-04
## stat22      -7.584984e-05
## stat23       3.405115e-04
## stat24      -4.068073e-04
## stat26      -3.201375e-05
## stat38       1.680958e-04
## stat39      -1.030395e-06
## stat41      -6.777415e-05
## stat42      -4.812949e-06
## stat45      -2.985358e-06
## stat49       2.149365e-06
## stat51       1.512878e-04
## stat59       2.092215e-05
## stat60       2.411083e-04
## stat65      -9.141130e-05
## stat84      -4.397408e-06
## stat87      -3.946556e-06
## stat89      -8.543456e-07
## stat91      -2.686681e-04
## stat98       3.249937e-03
## stat99       1.070522e-04
## stat100      1.766726e-04
## stat103     -1.696340e-04
## stat104     -5.099973e-05
## stat110     -3.154240e-03
## stat115      8.478076e-05
## stat128     -1.764039e-05
## stat130      9.390455e-06
## stat134     -7.305924e-05
## stat144      1.889752e-04
## stat146     -1.949772e-04
## stat149     -3.499702e-04
## stat156      7.923904e-05
## stat170     -1.441685e-04
## stat175     -6.344939e-06
## stat187     -1.492168e-04
## stat195      3.114049e-04
## stat198     -6.395941e-06
## stat204     -4.231580e-05
## stat207      3.040331e-05
## stat213     -3.257276e-05
## x18.sqrt     2.489591e-02

Test

if (algo.LASSO.caret == TRUE){
  test.model(model.LASSO.caret, data.test
             ,method = 'glmnet',subopt = "LASSO"
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.047   2.085   2.097   2.097   2.109   2.140 
## [1] "glmnet LASSO Test MSE: 0.000956496303164281"

LARS with CV

Train

if (algo.LARS.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "lars"
                                   ,subopt = 'NULL'
                                   ,feature.names = feature.names)
  model.LARS.caret = returned$model
}
## Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, : There were missing values in resampled
## performance measures.
## Aggregating results
## Selecting tuning parameters
## Fitting fraction = 0.384 on full training set
## Least Angle Regression 
## 
## 5584 samples
##  240 predictor
## 
## Pre-processing: centered (240), scaled (240) 
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 5026, 5026, 5026, 5025, 5025, 5026, ... 
## Resampling results across tuning parameters:
## 
##   fraction    RMSE        Rsquared   MAE       
##   0.00000000  0.03626493        NaN  0.02784941
##   0.01010101  0.03584682  0.1067604  0.02756173
##   0.02020202  0.03547553  0.1067604  0.02730797
##   0.03030303  0.03515324  0.1070568  0.02709579
##   0.04040404  0.03487380  0.1219918  0.02690395
##   0.05050505  0.03461981  0.1346689  0.02672919
##   0.06060606  0.03438517  0.1489220  0.02655464
##   0.07070707  0.03415730  0.1624803  0.02637883
##   0.08080808  0.03394089  0.1732640  0.02620913
##   0.09090909  0.03373693  0.1816206  0.02605002
##   0.10101010  0.03354565  0.1880599  0.02589794
##   0.11111111  0.03336727  0.1930085  0.02575128
##   0.12121212  0.03320445  0.1972600  0.02561124
##   0.13131313  0.03305133  0.2019991  0.02548294
##   0.14141414  0.03290771  0.2060252  0.02536112
##   0.15151515  0.03277470  0.2092740  0.02524431
##   0.16161616  0.03265246  0.2118833  0.02513350
##   0.17171717  0.03254109  0.2139674  0.02502805
##   0.18181818  0.03244483  0.2155209  0.02493374
##   0.19191919  0.03236321  0.2170739  0.02485266
##   0.20202020  0.03228842  0.2189184  0.02477694
##   0.21212121  0.03221610  0.2209353  0.02470310
##   0.22222222  0.03215353  0.2226320  0.02463734
##   0.23232323  0.03210185  0.2240580  0.02458397
##   0.24242424  0.03205642  0.2253866  0.02453799
##   0.25252525  0.03201520  0.2266664  0.02449587
##   0.26262626  0.03197679  0.2279650  0.02445619
##   0.27272727  0.03194251  0.2291366  0.02442190
##   0.28282828  0.03191280  0.2301533  0.02439186
##   0.29292929  0.03188811  0.2309605  0.02436614
##   0.30303030  0.03186772  0.2316110  0.02434337
##   0.31313131  0.03185084  0.2321452  0.02432385
##   0.32323232  0.03183670  0.2325810  0.02430739
##   0.33333333  0.03182594  0.2328777  0.02429458
##   0.34343434  0.03181698  0.2331088  0.02428401
##   0.35353535  0.03180960  0.2332838  0.02427558
##   0.36363636  0.03180464  0.2333533  0.02426850
##   0.37373737  0.03180180  0.2333315  0.02426271
##   0.38383838  0.03180102  0.2332133  0.02425893
##   0.39393939  0.03180135  0.2330537  0.02425632
##   0.40404040  0.03180231  0.2328777  0.02425462
##   0.41414141  0.03180407  0.2326740  0.02425333
##   0.42424242  0.03180634  0.2324582  0.02425248
##   0.43434343  0.03180908  0.2322308  0.02425214
##   0.44444444  0.03181224  0.2319909  0.02425281
##   0.45454545  0.03181611  0.2317234  0.02425417
##   0.46464646  0.03182059  0.2314342  0.02425570
##   0.47474747  0.03182569  0.2311214  0.02425773
##   0.48484848  0.03183138  0.2307888  0.02426029
##   0.49494949  0.03183768  0.2304321  0.02426351
##   0.50505051  0.03184419  0.2300719  0.02426718
##   0.51515152  0.03185099  0.2297054  0.02427126
##   0.52525253  0.03185857  0.2293074  0.02427615
##   0.53535354  0.03186610  0.2289194  0.02428113
##   0.54545455  0.03187393  0.2285238  0.02428649
##   0.55555556  0.03188208  0.2281188  0.02429231
##   0.56565657  0.03189044  0.2277095  0.02429860
##   0.57575758  0.03189878  0.2273075  0.02430483
##   0.58585859  0.03190725  0.2269045  0.02431112
##   0.59595960  0.03191620  0.2264837  0.02431771
##   0.60606061  0.03192537  0.2260576  0.02432471
##   0.61616162  0.03193470  0.2256289  0.02433193
##   0.62626263  0.03194414  0.2252007  0.02433904
##   0.63636364  0.03195381  0.2247676  0.02434629
##   0.64646465  0.03196347  0.2243403  0.02435337
##   0.65656566  0.03197317  0.2239159  0.02436045
##   0.66666667  0.03198340  0.2234718  0.02436791
##   0.67676768  0.03199362  0.2230324  0.02437542
##   0.68686869  0.03200381  0.2225982  0.02438283
##   0.69696970  0.03201429  0.2221544  0.02439039
##   0.70707071  0.03202496  0.2217053  0.02439803
##   0.71717172  0.03203546  0.2212690  0.02440567
##   0.72727273  0.03204594  0.2208382  0.02441349
##   0.73737374  0.03205655  0.2204059  0.02442148
##   0.74747475  0.03206722  0.2199755  0.02442962
##   0.75757576  0.03207791  0.2195487  0.02443769
##   0.76767677  0.03208869  0.2191226  0.02444576
##   0.77777778  0.03209983  0.2186850  0.02445415
##   0.78787879  0.03211127  0.2182380  0.02446269
##   0.79797980  0.03212293  0.2177849  0.02447133
##   0.80808081  0.03213500  0.2173175  0.02448043
##   0.81818182  0.03214753  0.2168341  0.02449008
##   0.82828283  0.03216043  0.2163390  0.02450025
##   0.83838384  0.03217357  0.2158381  0.02451047
##   0.84848485  0.03218680  0.2153378  0.02452064
##   0.85858586  0.03220010  0.2148400  0.02453090
##   0.86868687  0.03221338  0.2143479  0.02454105
##   0.87878788  0.03222689  0.2138506  0.02455130
##   0.88888889  0.03224059  0.2133485  0.02456164
##   0.89898990  0.03225460  0.2128376  0.02457222
##   0.90909091  0.03226853  0.2123342  0.02458282
##   0.91919192  0.03228229  0.2118431  0.02459336
##   0.92929293  0.03229603  0.2113568  0.02460415
##   0.93939394  0.03230986  0.2108722  0.02461510
##   0.94949495  0.03232371  0.2103911  0.02462617
##   0.95959596  0.03233772  0.2099078  0.02463734
##   0.96969697  0.03235179  0.2094257  0.02464851
##   0.97979798  0.03236590  0.2089462  0.02465970
##   0.98989899  0.03238008  0.2084683  0.02467118
##   1.00000000  0.03239445  0.2079861  0.02468299
## 
## RMSE was used to select the optimal model using the smallest value.
## The final value used for the model was fraction = 0.3838384.

##     fraction
## 39 0.3838384
## Warning: Removed 1 rows containing missing values (geom_point).

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients"
##            x4            x7            x8            x9           x10           x11           x14           x16 
## -1.503891e-03  6.972241e-03  7.350915e-04  3.471078e-03  8.852481e-04  1.195932e-04 -3.289865e-04  1.455005e-03 
##           x17           x21         stat3         stat4         stat8        stat13        stat14        stat20 
##  1.586320e-03  8.834458e-04  2.648042e-04 -1.249032e-04  3.547285e-05 -7.191729e-04 -9.146487e-04 -2.310368e-04 
##        stat22        stat23        stat24        stat26        stat38        stat41        stat51        stat59 
## -1.310376e-04  5.915317e-04 -7.028833e-04 -5.556684e-05  2.892139e-04 -1.183647e-04  2.612074e-04  3.586599e-05 
##        stat60        stat65        stat91        stat98        stat99       stat100       stat103       stat104 
##  4.157262e-04 -1.571473e-04 -4.672216e-04  5.718908e-03  1.836395e-04  3.072493e-04 -2.910198e-04 -8.895912e-05 
##       stat110       stat115       stat128       stat130       stat134       stat144       stat146       stat149 
## -5.465519e-03  1.463169e-04 -3.088174e-05  1.612893e-05 -1.269681e-04  3.277211e-04 -3.347786e-04 -6.022024e-04 
##       stat156       stat170       stat187       stat195       stat204       stat207       stat213      x18.sqrt 
##  1.359563e-04 -2.492273e-04 -2.593938e-04  5.424956e-04 -7.327397e-05  5.289702e-05 -5.563336e-05  1.124436e-02

Test

if (algo.LARS.caret == TRUE){
  test.model(model.LARS.caret, data.test
             ,method = 'lars',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.047   2.085   2.097   2.097   2.109   2.140 
## [1] "lars  Test MSE: 0.000956487756724553"

Session Info

sessionInfo()
## R version 3.5.2 (2018-12-20)
## Platform: x86_64-w64-mingw32/x64 (64-bit)
## Running under: Windows 10 x64 (build 17763)
## 
## Matrix products: default
## 
## locale:
## [1] LC_COLLATE=English_United States.1252  LC_CTYPE=English_United States.1252    LC_MONETARY=English_United States.1252
## [4] LC_NUMERIC=C                           LC_TIME=English_United States.1252    
## 
## attached base packages:
## [1] parallel  stats     graphics  grDevices utils     datasets  methods   base     
## 
## other attached packages:
##  [1] knitr_1.21                 htmltools_0.3.6            reshape2_1.4.3             lars_1.2                  
##  [5] doParallel_1.0.14          iterators_1.0.10           caret_6.0-81               leaps_3.0                 
##  [9] ggforce_0.1.3              rlist_0.4.6.1              car_3.0-2                  carData_3.0-2             
## [13] bestNormalize_1.3.0        scales_1.0.0               onewaytests_2.0            caTools_1.17.1.1          
## [17] mosaic_1.5.0               mosaicData_0.17.0          ggformula_0.9.1            ggstance_0.3.1            
## [21] lattice_0.20-38            DT_0.5                     ggiraphExtra_0.2.9         ggiraph_0.6.0             
## [25] investr_1.4.0              glmnet_2.0-16              foreach_1.4.4              Matrix_1.2-15             
## [29] MASS_7.3-51.1              PerformanceAnalytics_1.5.2 xts_0.11-2                 zoo_1.8-4                 
## [33] forcats_0.3.0              stringr_1.4.0              dplyr_0.8.0.1              purrr_0.3.0               
## [37] readr_1.3.1                tidyr_0.8.2                tibble_2.0.1               ggplot2_3.1.0             
## [41] tidyverse_1.2.1            usdm_1.1-18                raster_2.8-19              sp_1.3-1                  
## [45] pacman_0.5.0              
## 
## loaded via a namespace (and not attached):
##  [1] readxl_1.3.0       backports_1.1.3    plyr_1.8.4         lazyeval_0.2.1     splines_3.5.2      mycor_0.1.1       
##  [7] crosstalk_1.0.0    leaflet_2.0.2      digest_0.6.18      magrittr_1.5       mosaicCore_0.6.0   openxlsx_4.1.0    
## [13] recipes_0.1.4      modelr_0.1.3       gower_0.1.2        colorspace_1.4-0   rvest_0.3.2        ggrepel_0.8.0     
## [19] haven_2.0.0        xfun_0.4           crayon_1.3.4       jsonlite_1.6       survival_2.43-3    glue_1.3.0        
## [25] registry_0.5       gtable_0.2.0       ppcor_1.1          ipred_0.9-8        sjmisc_2.7.7       abind_1.4-5       
## [31] rngtools_1.3.1     bibtex_0.4.2       Rcpp_1.0.0         xtable_1.8-3       units_0.6-2        foreign_0.8-71    
## [37] stats4_3.5.2       lava_1.6.5         prodlim_2018.04.18 prediction_0.3.6.2 htmlwidgets_1.3    httr_1.4.0        
## [43] RColorBrewer_1.1-2 pkgconfig_2.0.2    farver_1.1.0       nnet_7.3-12        labeling_0.3       tidyselect_0.2.5  
## [49] rlang_0.3.1        later_0.8.0        munsell_0.5.0      cellranger_1.1.0   tools_3.5.2        cli_1.0.1         
## [55] generics_0.0.2     moments_0.14       sjlabelled_1.0.16  broom_0.5.1        evaluate_0.13      ggdendro_0.1-20   
## [61] yaml_2.2.0         ModelMetrics_1.2.2 zip_1.0.0          nlme_3.1-137       doRNG_1.7.1        mime_0.6          
## [67] xml2_1.2.0         compiler_3.5.2     rstudioapi_0.9.0   curl_3.3           tweenr_1.0.1       stringi_1.3.1     
## [73] highr_0.7          gdtools_0.1.7      stringdist_0.9.5.1 pillar_1.3.1       data.table_1.12.0  bitops_1.0-6      
## [79] httpuv_1.4.5.1     R6_2.4.0           promises_1.0.1     gridExtra_2.3      rio_0.5.16         codetools_0.2-15  
## [85] assertthat_0.2.0   pkgmaker_0.27      withr_2.1.2        nortest_1.0-4      mgcv_1.8-26        hms_0.4.2         
## [91] quadprog_1.5-5     grid_3.5.2         rpart_4.1-13       timeDate_3043.102  class_7.3-14       rmarkdown_1.11    
## [97] snakecase_0.9.2    shiny_1.2.0        lubridate_1.7.4